From 1629bcdb5da8963de38e6c8eef97fece99b480e6 Mon Sep 17 00:00:00 2001
From: Max Dexheimer <maxdexh03@gmail.com>
Date: Sun, 13 Sep 2026 18:30:48 +0200
Subject: [PATCH 01/28] Add useful APIs to `Unique(Arc|Rc)`

---
 library/alloc/src/rc.rs   | 85 +++++++++++++++++++++++++++++++-------
 library/alloc/src/sync.rs | 86 ++++++++++++++++++++++++++++++++-------
 2 files changed, 141 insertions(+), 30 deletions(-)

diff --git a/library/alloc/src/rc.rs b/library/alloc/src/rc.rs
index b2afe0b464eb6..eebbf7a5baed5 100644
--- a/library/alloc/src/rc.rs
+++ b/library/alloc/src/rc.rs
@@ -4326,6 +4326,13 @@ impl<T> UniqueRc<T> {
     pub fn new(value: T) -> Self {
         Self::new_in(value, Global)
     }
+
+    /// Like [`new`](Self::new), but returns an error if the allocation
+    /// fails, instead of calling [`handle_alloc_error`].
+    #[unstable(feature = "unique_rc_arc", issue = "112566")]
+    pub fn try_new(value: T) -> Result<Self, AllocError> {
+        Self::try_new_in(value, Global)
+    }
 }
 
 impl<T, A: Allocator> UniqueRc<T, A> {
@@ -4337,8 +4344,8 @@ impl<T, A: Allocator> UniqueRc<T, A> {
     /// point to the new [`Rc`].
     #[cfg(not(no_global_oom_handling))]
     #[unstable(feature = "unique_rc_arc", issue = "112566")]
-    #[must_use]
     // #[unstable(feature = "allocator_api", issue = "32838")]
+    #[must_use]
     pub fn new_in(value: T, alloc: A) -> Self {
         let (ptr, alloc) = Box::into_unique(Box::new_in(
             RcInner {
@@ -4353,8 +4360,29 @@ impl<T, A: Allocator> UniqueRc<T, A> {
         Self { ptr: ptr.into(), _marker: PhantomData, _marker2: PhantomData, alloc }
     }
 
-    #[cfg(not(no_global_oom_handling))]
-    fn unwrap_with_allocator(this: Self) -> (T, A) {
+    /// Like [`new_in`](Self::new_in), but returns an error if the allocation
+    /// fails, instead of calling [`handle_alloc_error`].
+    #[unstable(feature = "unique_rc_arc", issue = "112566")]
+    // #[unstable(feature = "allocator_api", issue = "32838")]
+    pub fn try_new_in(value: T, alloc: A) -> Result<Self, AllocError> {
+        let (ptr, alloc) = Box::into_non_null_with_allocator(Box::try_new_in(
+            RcInner {
+                strong: Cell::new(0),
+                // keep one weak reference so if all the weak pointers that are created are dropped
+                // the UniqueRc still stays valid.
+                weak: Cell::new(1),
+                value,
+            },
+            alloc,
+        )?);
+        Ok(Self { ptr, _marker: PhantomData, _marker2: PhantomData, alloc })
+    }
+
+    /// Consumes the `UniqueRc`, returning its wrapped value and allocator.
+    #[unstable(feature = "unique_rc_arc", issue = "112566")]
+    // #[unstable(feature = "allocator_api", issue = "32838")]
+    #[must_use]
+    pub fn unwrap_with_allocator(this: Self) -> (T, A) {
         let inner_ptr = this.ptr;
         let (data_ptr, alloc) = Self::into_raw_with_allocator(this);
 
@@ -4368,6 +4396,13 @@ impl<T, A: Allocator> UniqueRc<T, A> {
         (val, alloc)
     }
 
+    /// Consumes the `UniqueRc`, returning its wrapped value.
+    #[unstable(feature = "unique_rc_arc", issue = "112566")]
+    #[must_use]
+    pub fn unwrap(this: Self) -> T {
+        Self::unwrap_with_allocator(this).0
+    }
+
     /// Maps the value in a `UniqueRc`, reusing the allocation if possible.
     ///
     /// `f` is called on a reference to the value in the `UniqueRc`, and the result is returned,
@@ -4399,11 +4434,10 @@ impl<T, A: Allocator> UniqueRc<T, A> {
             unsafe {
                 let (ptr, alloc) = UniqueRc::into_raw_with_allocator(this);
                 let value = ptr.read();
-                let mut allocation =
+                let allocation =
                     UniqueRc::from_raw_with_allocator(ptr.cast::<mem::MaybeUninit<U>>(), alloc);
 
-                allocation.write(f(value));
-                allocation.assume_init()
+                UniqueRc::write(allocation, f(value))
             }
         } else {
             let (val, alloc) = UniqueRc::unwrap_with_allocator(this);
@@ -4450,13 +4484,12 @@ impl<T, A: Allocator> UniqueRc<T, A> {
             unsafe {
                 let (ptr, alloc) = UniqueRc::into_raw_with_allocator(this);
                 let value = ptr.read();
-                let mut allocation = UniqueRc::from_raw_with_allocator(
+                let allocation = UniqueRc::from_raw_with_allocator(
                     ptr.cast::<mem::MaybeUninit<R::Output>>(),
                     alloc,
                 );
 
-                allocation.write(f(value)?);
-                try { allocation.assume_init() }
+                try { UniqueRc::write(allocation, f(value)?) }
             }
         } else {
             let (val, alloc) = UniqueRc::unwrap_with_allocator(this);
@@ -4484,7 +4517,6 @@ impl<T: ?Sized, A: Allocator> UniqueRc<T, A> {
         }
     }
 
-    #[cfg(not(no_global_oom_handling))]
     fn into_raw_with_allocator(this: Self) -> (*const T, A) {
         let this = ManuallyDrop::new(this);
         // SAFETY: The copy of the allocator stored in `this` is forgotten
@@ -4526,7 +4558,6 @@ impl<T: ?Sized, A: Allocator> UniqueRc<T, A> {
         unsafe { self.ptr.as_ref() }
     }
 
-    #[cfg(not(no_global_oom_handling))]
     fn as_ptr(this: &Self) -> *const T {
         let ptr: *mut RcInner<T> = NonNull::as_ptr(this.ptr);
 
@@ -4537,7 +4568,6 @@ impl<T: ?Sized, A: Allocator> UniqueRc<T, A> {
     }
 
     #[inline]
-    #[cfg(not(no_global_oom_handling))]
     fn into_inner_with_allocator(this: Self) -> (NonNull<RcInner<T>>, A) {
         let this = mem::ManuallyDrop::new(this);
         // SAFETY: Pointer is valid for reads.
@@ -4545,7 +4575,6 @@ impl<T: ?Sized, A: Allocator> UniqueRc<T, A> {
     }
 
     #[inline]
-    #[cfg(not(no_global_oom_handling))]
     unsafe fn from_inner_in(ptr: NonNull<RcInner<T>>, alloc: A) -> Self {
         Self { ptr, _marker: PhantomData, _marker2: PhantomData, alloc }
     }
@@ -4567,9 +4596,35 @@ impl<T: ?Sized, A: AllocatorClone> UniqueRc<T, A> {
     }
 }
 
-#[cfg(not(no_global_oom_handling))]
 impl<T, A: Allocator> UniqueRc<mem::MaybeUninit<T>, A> {
-    unsafe fn assume_init(self) -> UniqueRc<T, A> {
+    /// Writes the value and converts to `UniqueRc<T, A>`.
+    ///
+    /// This method converts similarly to [`assume_init`](Self::assume_init) but
+    /// writes `value` into it before conversion, thus guaranteeing safety.
+    #[unstable(feature = "unique_rc_arc", issue = "112566")]
+    #[must_use]
+    pub fn write(mut this: Self, value: T) -> UniqueRc<T, A> {
+        // SAFETY: Writing initialises the wrapped value.
+        unsafe {
+            this.write(value);
+            this.assume_init()
+        }
+    }
+
+    /// Converts to `UniqueRc<T, A>`.
+    ///
+    /// # Safety
+    ///
+    /// As with [`MaybeUninit::assume_init`],
+    /// it is up to the caller to guarantee that the value
+    /// really is in an initialized state.
+    /// Calling this when the content is not yet fully initialized
+    /// causes immediate undefined behavior.
+    ///
+    /// [`MaybeUninit::assume_init`]: mem::MaybeUninit::assume_init
+    #[unstable(feature = "unique_rc_arc", issue = "112566")]
+    #[must_use]
+    pub unsafe fn assume_init(self) -> UniqueRc<T, A> {
         let (ptr, alloc) = UniqueRc::into_inner_with_allocator(self);
         // SAFETY: Upheld by caller.
         unsafe { UniqueRc::from_inner_in(ptr.cast(), alloc) }
diff --git a/library/alloc/src/sync.rs b/library/alloc/src/sync.rs
index 09a371f94bbb9..343ab05f68bc6 100644
--- a/library/alloc/src/sync.rs
+++ b/library/alloc/src/sync.rs
@@ -4803,6 +4803,13 @@ impl<T> UniqueArc<T, Global> {
     pub fn new(value: T) -> Self {
         Self::new_in(value, Global)
     }
+
+    /// Like [`new`](Self::new), but returns an error if the allocation
+    /// fails, instead of calling [`handle_alloc_error`].
+    #[unstable(feature = "unique_rc_arc", issue = "112566")]
+    pub fn try_new(value: T) -> Result<Self, AllocError> {
+        Self::try_new_in(value, Global)
+    }
 }
 
 impl<T, A: Allocator> UniqueArc<T, A> {
@@ -4814,8 +4821,8 @@ impl<T, A: Allocator> UniqueArc<T, A> {
     /// point to the new [`Arc`].
     #[cfg(not(no_global_oom_handling))]
     #[unstable(feature = "unique_rc_arc", issue = "112566")]
-    #[must_use]
     // #[unstable(feature = "allocator_api", issue = "32838")]
+    #[must_use]
     pub fn new_in(data: T, alloc: A) -> Self {
         let (ptr, alloc) = Box::into_unique(Box::new_in(
             ArcInner {
@@ -4830,8 +4837,29 @@ impl<T, A: Allocator> UniqueArc<T, A> {
         Self { ptr: ptr.into(), _marker: PhantomData, _marker2: PhantomData, alloc }
     }
 
-    #[cfg(not(no_global_oom_handling))]
-    fn unwrap_with_allocator(this: Self) -> (T, A) {
+    /// Like [`new_in`](Self::new_in), but returns an error if the allocation
+    /// fails, instead of calling [`handle_alloc_error`].
+    #[unstable(feature = "unique_rc_arc", issue = "112566")]
+    // #[unstable(feature = "allocator_api", issue = "32838")]
+    pub fn try_new_in(data: T, alloc: A) -> Result<Self, AllocError> {
+        let (ptr, alloc) = Box::into_non_null_with_allocator(Box::try_new_in(
+            ArcInner {
+                strong: atomic::AtomicUsize::new(0),
+                // keep one weak reference so if all the weak pointers that are created are dropped
+                // the UniqueArc still stays valid.
+                weak: atomic::AtomicUsize::new(1),
+                data,
+            },
+            alloc,
+        )?);
+        Ok(Self { ptr, _marker: PhantomData, _marker2: PhantomData, alloc })
+    }
+
+    /// Consumes the `UniqueArc`, returning its wrapped value and allocator.
+    #[unstable(feature = "unique_rc_arc", issue = "112566")]
+    // #[unstable(feature = "allocator_api", issue = "32838")]
+    #[must_use]
+    pub fn unwrap_with_allocator(this: Self) -> (T, A) {
         let inner_ptr = this.ptr;
         let (data_ptr, alloc) = Self::into_raw_with_allocator(this);
 
@@ -4839,11 +4867,19 @@ impl<T, A: Allocator> UniqueArc<T, A> {
         // We do not use the data inside ever again.
         let val = unsafe { data_ptr.read() };
 
+        // Drop the strong-weak ref
         drop(Weak { ptr: inner_ptr, alloc: &alloc });
 
         (val, alloc)
     }
 
+    /// Consumes the `UniqueArc`, returning its wrapped value.
+    #[unstable(feature = "unique_rc_arc", issue = "112566")]
+    #[must_use]
+    pub fn unwrap(this: Self) -> T {
+        Self::unwrap_with_allocator(this).0
+    }
+
     /// Maps the value in a `UniqueArc`, reusing the allocation if possible.
     ///
     /// `f` is called on a reference to the value in the `UniqueArc`, and the result is returned,
@@ -4875,11 +4911,10 @@ impl<T, A: Allocator> UniqueArc<T, A> {
             unsafe {
                 let (ptr, alloc) = UniqueArc::into_raw_with_allocator(this);
                 let value = ptr.read();
-                let mut allocation =
+                let allocation =
                     UniqueArc::from_raw_with_allocator(ptr.cast::<mem::MaybeUninit<U>>(), alloc);
 
-                allocation.write(f(value));
-                allocation.assume_init()
+                UniqueArc::write(allocation, f(value))
             }
         } else {
             let (val, alloc) = UniqueArc::unwrap_with_allocator(this);
@@ -4926,13 +4961,12 @@ impl<T, A: Allocator> UniqueArc<T, A> {
             unsafe {
                 let (ptr, alloc) = UniqueArc::into_raw_with_allocator(this);
                 let value = ptr.read();
-                let mut allocation = UniqueArc::from_raw_with_allocator(
+                let allocation = UniqueArc::from_raw_with_allocator(
                     ptr.cast::<mem::MaybeUninit<R::Output>>(),
                     alloc,
                 );
 
-                allocation.write(f(value)?);
-                try { allocation.assume_init() }
+                try { UniqueArc::write(allocation, f(value)?) }
             }
         } else {
             let (val, alloc) = UniqueArc::unwrap_with_allocator(this);
@@ -4960,7 +4994,6 @@ impl<T: ?Sized, A: Allocator> UniqueArc<T, A> {
         }
     }
 
-    #[cfg(not(no_global_oom_handling))]
     fn into_raw_with_allocator(this: Self) -> (*const T, A) {
         let this = ManuallyDrop::new(this);
         // SAFETY: The copy of the allocator stored in `this` is forgotten
@@ -5003,7 +5036,6 @@ impl<T: ?Sized, A: Allocator> UniqueArc<T, A> {
         unsafe { self.ptr.as_ref() }
     }
 
-    #[cfg(not(no_global_oom_handling))]
     fn as_ptr(this: &Self) -> *const T {
         let ptr: *mut ArcInner<T> = NonNull::as_ptr(this.ptr);
 
@@ -5014,7 +5046,6 @@ impl<T: ?Sized, A: Allocator> UniqueArc<T, A> {
     }
 
     #[inline]
-    #[cfg(not(no_global_oom_handling))]
     fn into_inner_with_allocator(this: Self) -> (NonNull<ArcInner<T>>, A) {
         let this = mem::ManuallyDrop::new(this);
         // SAFETY: Pointer is valid for reads and only read once.
@@ -5022,7 +5053,6 @@ impl<T: ?Sized, A: Allocator> UniqueArc<T, A> {
     }
 
     #[inline]
-    #[cfg(not(no_global_oom_handling))]
     unsafe fn from_inner_in(ptr: NonNull<ArcInner<T>>, alloc: A) -> Self {
         Self { ptr, _marker: PhantomData, _marker2: PhantomData, alloc }
     }
@@ -5056,9 +5086,35 @@ impl<T: ?Sized, A: AllocatorClone> UniqueArc<T, A> {
     }
 }
 
-#[cfg(not(no_global_oom_handling))]
 impl<T, A: Allocator> UniqueArc<mem::MaybeUninit<T>, A> {
-    unsafe fn assume_init(self) -> UniqueArc<T, A> {
+    /// Writes the value and converts to `UniqueArc<T, A>`.
+    ///
+    /// This method converts similarly to [`assume_init`](Self::assume_init) but
+    /// writes `value` into it before conversion, thus guaranteeing safety.
+    #[unstable(feature = "unique_rc_arc", issue = "112566")]
+    #[must_use]
+    pub fn write(mut this: Self, value: T) -> UniqueArc<T, A> {
+        // SAFETY: Writing initialises the wrapped value.
+        unsafe {
+            this.write(value);
+            this.assume_init()
+        }
+    }
+
+    /// Converts to `UniqueArc<T, A>`.
+    ///
+    /// # Safety
+    ///
+    /// As with [`MaybeUninit::assume_init`],
+    /// it is up to the caller to guarantee that the value
+    /// really is in an initialized state.
+    /// Calling this when the content is not yet fully initialized
+    /// causes immediate undefined behavior.
+    ///
+    /// [`MaybeUninit::assume_init`]: mem::MaybeUninit::assume_init
+    #[unstable(feature = "unique_rc_arc", issue = "112566")]
+    #[must_use]
+    pub unsafe fn assume_init(self) -> UniqueArc<T, A> {
         let (ptr, alloc) = UniqueArc::into_inner_with_allocator(self);
         // SAFETY: Upheld by caller.
         unsafe { UniqueArc::from_inner_in(ptr.cast(), alloc) }

From a1864bdeb0c5252093ca7948f83d793ef712f555 Mon Sep 17 00:00:00 2001
From: ltdk <usr@ltdk.xyz>
Date: Tue, 15 Sep 2026 23:07:58 -0400
Subject: [PATCH 02/28] Ping T-libs-ping instead of T-libs-fcp for backports

---
 triagebot.toml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/triagebot.toml b/triagebot.toml
index fc9c43d2dbcae..67799f0a944ef 100644
--- a/triagebot.toml
+++ b/triagebot.toml
@@ -850,7 +850,7 @@ zulip_stream = 542373 # #t-libs/backports
 topic = "#{number}: beta-nominated"
 message_on_add = [
     """\
-@*T-libs-fcp* PR #{number} "{title}" has been nominated for beta backport.
+@*T-libs-ping* PR #{number} "{title}" has been nominated for beta backport.
 """,
     """\
 /poll Should #{number} be beta backported?
@@ -874,7 +874,7 @@ zulip_stream = 542373 # #t-libs/backports
 topic = "#{number}: stable-nominated"
 message_on_add = [
     """\
-@*T-libs-fcp* PR #{number} "{title}" has been nominated for stable backport.
+@*T-libs-ping* PR #{number} "{title}" has been nominated for stable backport.
 """,
     """\
 /poll Approve stable backport of #{number}?

From 085678ad35d740494b98aa9774bae60ce824550b Mon Sep 17 00:00:00 2001
From: Flakebi <flakebi@t-online.de>
Date: Tue, 1 Sep 2026 09:40:37 +0200
Subject: [PATCH 03/28] Add address_space and byref to abi PassMode::Indirect
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Both will be used by the amdgpu target to implement the `gpu-kernel`
ABI.

`address_space` specifies the address space of an indirect argument.

`AmdgpuKernelArg` translates to LLVM’s byref, which is similar to
on_stack/byval, however, there is no extra copy made, the pointer may
not point to the stack but can point to some other address space, and
the passed argument should not be modified.

byval and byref are mutually exclusive, so change on_stack to an enum
with the new states, Pointer (none), OnStack and AmdgpuKernelArg.
---
 compiler/rustc_abi/src/layout/ty.rs           |   4 +-
 .../src/abi/pass_mode.rs                      |  24 ++--
 .../src/abi/returning.rs                      |  17 +--
 compiler/rustc_codegen_gcc/src/abi.rs         |  33 ++++-
 compiler/rustc_codegen_llvm/src/abi.rs        | 111 +++++++++++++---
 compiler/rustc_codegen_llvm/src/llvm/ffi.rs   |   1 +
 compiler/rustc_codegen_llvm/src/llvm/mod.rs   |   4 +
 compiler/rustc_codegen_ssa/src/mir/block.rs   |  48 ++++---
 compiler/rustc_codegen_ssa/src/mir/mod.rs     |  16 ++-
 .../rustc_llvm/llvm-wrapper/RustWrapper.cpp   |   5 +
 .../src/deduce_param_attrs.rs                 |   2 +-
 compiler/rustc_public/src/abi.rs              |  18 ++-
 .../src/unstable/convert/stable/abi.rs        |  35 +++--
 compiler/rustc_target/src/callconv/mod.rs     | 125 ++++++++++++++----
 compiler/rustc_target/src/callconv/x86.rs     |   5 +-
 compiler/rustc_target/src/callconv/xtensa.rs  |   6 +-
 compiler/rustc_ty_utils/src/abi.rs            |  14 +-
 tests/assembly-llvm/tail-call-indirect.rs     |   6 +-
 tests/ui-fulldeps/rustc_public/check_abi.rs   |   8 +-
 .../rustc_public/check_abi_cast.rs            |   4 +-
 tests/ui/abi/c-zst.powerpc-linux.stderr       |   3 +-
 tests/ui/abi/c-zst.s390x-linux.stderr         |   3 +-
 tests/ui/abi/c-zst.sparc64-linux.stderr       |   3 +-
 .../ui/abi/c-zst.x86_64-pc-windows-gnu.stderr |   3 +-
 tests/ui/abi/debug.generic.stderr             |   6 +-
 tests/ui/abi/debug.loongarch64.stderr         |   6 +-
 tests/ui/abi/debug.riscv64.stderr             |   6 +-
 tests/ui/abi/pass-indirectly-attr.rs          |   2 +-
 tests/ui/abi/pass-indirectly-attr.stderr      |   3 +-
 .../pass-by-value-abi.aarch64.stderr          |   3 +-
 tests/ui/c-variadic/pass-by-value-abi.rs      |   8 +-
 .../pass-by-value-abi.x86_64.stderr           |   9 +-
 tests/ui/explicit-tail-calls/indirect.rs      |  10 +-
 33 files changed, 402 insertions(+), 149 deletions(-)

diff --git a/compiler/rustc_abi/src/layout/ty.rs b/compiler/rustc_abi/src/layout/ty.rs
index 3b3a58697b205..54dede083f595 100644
--- a/compiler/rustc_abi/src/layout/ty.rs
+++ b/compiler/rustc_abi/src/layout/ty.rs
@@ -253,8 +253,8 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
     }
 
     /// If this method returns `true`, then this type should always have a `PassMode` of
-    /// `Indirect { on_stack: false, .. }` when being used as the argument type of a function with a
-    /// non-Rustic ABI (this is true for structs annotated with the
+    /// `Indirect { mode: IndirectMode::Pointer, .. }` when being used as the argument type of a
+    /// function with a non-Rustic ABI (this is true for structs annotated with the
     /// `#[rustc_pass_indirectly_in_non_rustic_abis]` attribute).
     ///
     /// This is used to replicate some of the behaviour of C array-to-pointer decay; however unlike
diff --git a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
index 1c552ca1a9c32..48ffc43c5cfa1 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
@@ -3,7 +3,7 @@
 use cranelift_codegen::ir::ArgumentPurpose;
 use rustc_abi::{Reg, RegKind};
 use rustc_target::callconv::{
-    ArgAbi, ArgAttributes, ArgExtension as RustcArgExtension, CastTarget, PassMode,
+    ArgAbi, ArgAttributes, ArgExtension as RustcArgExtension, CastTarget, IndirectMode, PassMode,
 };
 use smallvec::{SmallVec, smallvec};
 
@@ -126,8 +126,12 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
                 assert_eq!(pad_i32_count, 0, "padding support not yet implemented");
                 cast_target_to_abi_params(cast).into_iter().map(|(_, param)| param).collect()
             }
-            PassMode::Indirect { attrs, meta_attrs: None, on_stack } => {
-                if on_stack {
+            PassMode::Indirect { attrs, meta_attrs: None, address_space: _, mode } => {
+                assert!(
+                    mode != IndirectMode::AmdgpuKernelArg,
+                    "unsupported amdgpu kernel argument"
+                );
+                if mode == IndirectMode::OnStack {
                     // Abi requires aligning struct size to pointer size
                     let size = self.layout.size.align_to(tcx.data_layout.pointer_align().abi);
                     let size = u32::try_from(size.bytes()).unwrap();
@@ -139,8 +143,8 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
                     smallvec![apply_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), attrs)]
                 }
             }
-            PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => {
-                assert!(!on_stack);
+            PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), address_space: _, mode } => {
+                assert!(mode == IndirectMode::Pointer);
                 smallvec![
                     apply_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), attrs),
                     apply_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), meta_attrs),
@@ -184,8 +188,8 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
                 None,
                 cast_target_to_abi_params(cast).into_iter().map(|(_, param)| param).collect(),
             ),
-            PassMode::Indirect { attrs, meta_attrs: None, on_stack } => {
-                assert!(!on_stack);
+            PassMode::Indirect { attrs, meta_attrs: None, address_space: _, mode } => {
+                assert!(mode == IndirectMode::Pointer);
                 (
                     Some(apply_attrs_to_abi_param(
                         AbiParam::special(pointer_ty(tcx), ArgumentPurpose::StructReturn),
@@ -194,7 +198,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
                     vec![],
                 )
             }
-            PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+            PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
                 unreachable!("unsized return value")
             }
         }
@@ -324,7 +328,7 @@ pub(super) fn cvalue_for_param<'tcx>(
         PassMode::Cast { ref cast, .. } => {
             from_casted_value(fx, &block_params, arg_abi.layout, cast)
         }
-        PassMode::Indirect { attrs, meta_attrs: None, on_stack: _ } => {
+        PassMode::Indirect { attrs, meta_attrs: None, address_space: _, mode: _ } => {
             assert_eq!(block_params.len(), 1, "{:?}", block_params);
             if let Some(pointee_align) = attrs.pointee_align
                 && pointee_align < arg_abi.layout.align.abi
@@ -342,7 +346,7 @@ pub(super) fn cvalue_for_param<'tcx>(
                 CValue::by_ref(Pointer::new(block_params[0]), arg_abi.layout)
             }
         }
-        PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+        PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
             assert_eq!(block_params.len(), 2, "{:?}", block_params);
             CValue::by_ref_unsized(Pointer::new(block_params[0]), block_params[1], arg_abi.layout)
         }
diff --git a/compiler/rustc_codegen_cranelift/src/abi/returning.rs b/compiler/rustc_codegen_cranelift/src/abi/returning.rs
index 36087f96dd776..7f4ee9435b506 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/returning.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/returning.rs
@@ -17,12 +17,12 @@ pub(super) fn codegen_return_param<'tcx>(
             let is_ssa = ssa_analyzed[RETURN_PLACE].is_ssa(fx, fx.fn_abi.ret.layout.ty);
             (super::make_local_place(fx, RETURN_PLACE, fx.fn_abi.ret.layout, is_ssa), smallvec![])
         }
-        PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => {
+        PassMode::Indirect { attrs: _, meta_attrs: None, address_space: _, mode: _ } => {
             let ret_param = block_params_iter.next().unwrap();
             assert_eq!(fx.bcx.func.dfg.value_type(ret_param), fx.pointer_type);
             (CPlace::for_ptr(Pointer::new(ret_param), fx.fn_abi.ret.layout), smallvec![ret_param])
         }
-        PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+        PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
             unreachable!("unsized return value")
         }
     };
@@ -50,7 +50,7 @@ pub(super) fn codegen_with_call_return_arg<'tcx>(
 ) {
     let (ret_temp_place, return_ptr) = match ret_arg_abi.mode {
         PassMode::Ignore => (None, None),
-        PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => {
+        PassMode::Indirect { attrs: _, meta_attrs: None, address_space: _, mode: _ } => {
             if let Some(ret_ptr) = ret_place.try_to_ptr() {
                 // This is an optimization to prevent unnecessary copies of the return value when
                 // the return place is already a memory place as opposed to a register.
@@ -61,7 +61,7 @@ pub(super) fn codegen_with_call_return_arg<'tcx>(
                 (Some(place), Some(place.to_ptr().get_addr(fx)))
             }
         }
-        PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+        PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
             unreachable!("unsized return value")
         }
         PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast { .. } => (None, None),
@@ -86,14 +86,14 @@ pub(super) fn codegen_with_call_return_arg<'tcx>(
                 super::pass_mode::from_casted_value(fx, &results, ret_place.layout(), cast);
             ret_place.write_cvalue(fx, result);
         }
-        PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => {
+        PassMode::Indirect { attrs: _, meta_attrs: None, address_space: _, mode: _ } => {
             if let Some(ret_temp_place) = ret_temp_place {
                 // If ret_temp_place is None, it is not necessary to copy the return value.
                 let ret_temp_value = ret_temp_place.to_cvalue(fx);
                 ret_place.write_cvalue(fx, ret_temp_value);
             }
         }
-        PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+        PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
             unreachable!("unsized return value")
         }
     }
@@ -102,10 +102,11 @@ pub(super) fn codegen_with_call_return_arg<'tcx>(
 /// Codegen a return instruction with the right return value(s) if any.
 pub(crate) fn codegen_return(fx: &mut FunctionCx<'_, '_, '_>) {
     match fx.fn_abi.ret.mode {
-        PassMode::Ignore | PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => {
+        PassMode::Ignore
+        | PassMode::Indirect { attrs: _, meta_attrs: None, address_space: _, mode: _ } => {
             fx.bcx.ins().return_(&[]);
         }
-        PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+        PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
             unreachable!("unsized return value")
         }
         PassMode::Direct(_) => {
diff --git a/compiler/rustc_codegen_gcc/src/abi.rs b/compiler/rustc_codegen_gcc/src/abi.rs
index 6a05f1cbbeef1..b5834ca57ebe1 100644
--- a/compiler/rustc_codegen_gcc/src/abi.rs
+++ b/compiler/rustc_codegen_gcc/src/abi.rs
@@ -11,7 +11,7 @@ use rustc_middle::ty::layout::LayoutOf;
 #[cfg(feature = "master")]
 use rustc_session::{Session, config};
 use rustc_span::bug;
-use rustc_target::callconv::{ArgAttributes, CastTarget, FnAbi, PassMode};
+use rustc_target::callconv::{ArgAttributes, CastTarget, FnAbi, IndirectMode, PassMode};
 #[cfg(feature = "master")]
 use rustc_target::spec::Arch;
 
@@ -178,19 +178,42 @@ impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     let ty = cast.gcc_type(cx);
                     apply_attrs(ty, &cast.attrs, argument_tys.len())
                 }
-                PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: true } => {
+                PassMode::Indirect {
+                    attrs: _,
+                    meta_attrs: None,
+                    address_space: _,
+                    mode: IndirectMode::OnStack,
+                } => {
                     // This is a "byval" argument, so we don't apply the `restrict` attribute on it.
                     on_stack_param_indices.insert(argument_tys.len());
                     arg.layout.gcc_type(cx)
                 }
+                PassMode::Indirect {
+                    attrs: _,
+                    meta_attrs: None,
+                    address_space: _,
+                    mode: IndirectMode::AmdgpuKernelArg,
+                } => {
+                    unimplemented!("unsupported amdgpu kernel argument")
+                }
                 PassMode::Direct(attrs) => {
                     apply_attrs(arg.layout.immediate_gcc_type(cx), &attrs, argument_tys.len())
                 }
-                PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => {
+                PassMode::Indirect {
+                    attrs,
+                    meta_attrs: None,
+                    address_space: _,
+                    mode: IndirectMode::Pointer,
+                } => {
                     apply_attrs(cx.type_ptr_to(arg.layout.gcc_type(cx)), &attrs, argument_tys.len())
                 }
-                PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => {
-                    assert!(!on_stack);
+                PassMode::Indirect {
+                    attrs,
+                    meta_attrs: Some(meta_attrs),
+                    address_space: _,
+                    mode,
+                } => {
+                    assert!(mode == IndirectMode::Pointer);
                     // Construct the type of a (wide) pointer to `ty`, and pass its two fields.
                     // Any two ABI-compatible unsized types have the same metadata type and
                     // moreover the same metadata value leads to the same dynamic size and
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs
index a45138849e4e0..703986fdab3fa 100644
--- a/compiler/rustc_codegen_llvm/src/abi.rs
+++ b/compiler/rustc_codegen_llvm/src/abi.rs
@@ -15,7 +15,7 @@ use rustc_middle::ty::layout::LayoutOf;
 use rustc_session::{Session, config};
 use rustc_span::bug;
 use rustc_target::callconv::{
-    ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, CastTarget, FnAbi, PassMode,
+    ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, CastTarget, FnAbi, IndirectMode, PassMode,
 };
 use rustc_target::spec::{Arch, SanitizerSet};
 use smallvec::SmallVec;
@@ -242,12 +242,12 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
         match &self.mode {
             PassMode::Ignore => {}
             // Sized indirect arguments
-            PassMode::Indirect { attrs, meta_attrs: None, on_stack: _ } => {
+            PassMode::Indirect { attrs, meta_attrs: None, address_space: _, mode: _ } => {
                 let align = attrs.pointee_align.unwrap_or(self.layout.align.abi);
                 OperandValue::Ref(PlaceValue::new_sized(val, align)).store(bx, dst);
             }
             // Unsized indirect arguments cannot be stored
-            PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+            PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
                 bug!("unsized `ArgAbi` cannot be stored");
             }
             PassMode::Cast { cast, pad_i32_count: _ } => {
@@ -303,11 +303,11 @@ impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
             PassMode::Pair(..) => {
                 OperandValue::Pair(next(), next()).store(bx, dst);
             }
-            PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+            PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
                 bug!("unsized `ArgAbi` cannot be stored");
             }
             PassMode::Direct(_)
-            | PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ }
+            | PassMode::Indirect { attrs: _, meta_attrs: None, address_space: _, mode: _ }
             | PassMode::Cast { .. } => {
                 let next_arg = next();
                 self.store(bx, next_arg, dst);
@@ -368,8 +368,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
             PassMode::Ignore => cx.type_void(),
             PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx),
             PassMode::Cast { cast, pad_i32_count: _ } => cast.llvm_type(cx),
-            PassMode::Indirect { .. } => {
-                llargument_tys.push(cx.type_ptr());
+            PassMode::Indirect { address_space, .. } => {
+                let ty = if let Some(address_space) = address_space {
+                    cx.type_ptr_ext(*address_space)
+                } else {
+                    cx.type_ptr()
+                };
+                llargument_tys.push(ty);
                 cx.type_void()
             }
         };
@@ -394,7 +399,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 1, true));
                     continue;
                 }
-                PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+                PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
                     // Construct the type of a (wide) pointer to `ty`, and pass its two fields.
                     // Any two ABI-compatible unsized types have the same metadata type and
                     // moreover the same metadata value leads to the same dynamic size and
@@ -405,7 +410,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 1, true));
                     continue;
                 }
-                PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => cx.type_ptr(),
+                PassMode::Indirect { attrs: _, meta_attrs: None, address_space, mode: _ } => {
+                    if let Some(address_space) = address_space {
+                        cx.type_ptr_ext(*address_space)
+                    } else {
+                        cx.type_ptr()
+                    }
+                }
                 PassMode::Cast { cast, pad_i32_count } => {
                     // Add padding.
                     llargument_tys.extend(std::iter::repeat_n(
@@ -495,8 +506,8 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     apply_range_attr(llvm::AttributePlace::ReturnValue, scalar);
                 }
             }
-            PassMode::Indirect { attrs, meta_attrs: _, on_stack } => {
-                assert!(!on_stack);
+            PassMode::Indirect { attrs, meta_attrs: _, address_space: _, mode } => {
+                assert!(*mode == IndirectMode::Pointer);
                 let i = apply(attrs);
                 let sret = llvm::CreateStructRetAttr(
                     cx.llcx,
@@ -522,7 +533,12 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
         for arg in self.args.iter() {
             match &arg.mode {
                 PassMode::Ignore => {}
-                PassMode::Indirect { attrs, meta_attrs: None, on_stack: true } => {
+                PassMode::Indirect {
+                    attrs,
+                    meta_attrs: None,
+                    address_space: _,
+                    mode: IndirectMode::OnStack,
+                } => {
                     let i = apply(attrs);
                     let byval = llvm::CreateByValAttr(
                         cx.llcx,
@@ -530,13 +546,31 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     );
                     attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[byval]);
                 }
+                PassMode::Indirect {
+                    attrs,
+                    meta_attrs: None,
+                    address_space: _,
+                    mode: IndirectMode::AmdgpuKernelArg,
+                } => {
+                    let i = apply(attrs);
+                    let byref = llvm::CreateByRefAttr(
+                        cx.llcx,
+                        cx.type_array(cx.type_i8(), arg.layout.size.bytes()),
+                    );
+                    attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[byref]);
+                }
                 PassMode::Direct(attrs) => {
                     let i = apply(attrs);
                     if let BackendRepr::Scalar(scalar) = arg.layout.backend_repr {
                         apply_range_attr(llvm::AttributePlace::Argument(i), scalar);
                     }
                 }
-                PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => {
+                PassMode::Indirect {
+                    attrs,
+                    meta_attrs: None,
+                    address_space: _,
+                    mode: IndirectMode::Pointer,
+                } => {
                     let i = apply(attrs);
                     if cx.sess().opts.optimize != config::OptLevel::No {
                         attributes::apply_to_llfn(
@@ -546,8 +580,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                         );
                     }
                 }
-                PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => {
-                    assert!(!on_stack);
+                PassMode::Indirect {
+                    attrs,
+                    meta_attrs: Some(meta_attrs),
+                    address_space: _,
+                    mode,
+                } => {
+                    assert!(*mode == IndirectMode::Pointer);
                     apply(attrs);
                     apply(meta_attrs);
                 }
@@ -625,8 +664,8 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
             PassMode::Direct(attrs) => {
                 attrs.apply_attrs_to_callsite(llvm::AttributePlace::ReturnValue, bx.cx, callsite);
             }
-            PassMode::Indirect { attrs, meta_attrs: _, on_stack } => {
-                assert!(!on_stack);
+            PassMode::Indirect { attrs, meta_attrs: _, address_space: _, mode } => {
+                assert!(*mode == IndirectMode::Pointer);
                 let i = apply(bx.cx, attrs);
                 let sret = llvm::CreateStructRetAttr(
                     bx.cx.llcx,
@@ -646,7 +685,12 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
         for arg in self.args.iter() {
             match &arg.mode {
                 PassMode::Ignore => {}
-                PassMode::Indirect { attrs, meta_attrs: None, on_stack: true } => {
+                PassMode::Indirect {
+                    attrs,
+                    meta_attrs: None,
+                    address_space: _,
+                    mode: IndirectMode::OnStack,
+                } => {
                     let i = apply(bx.cx, attrs);
                     let byval = llvm::CreateByValAttr(
                         bx.cx.llcx,
@@ -658,11 +702,38 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                         &[byval],
                     );
                 }
+                PassMode::Indirect {
+                    attrs,
+                    meta_attrs: None,
+                    address_space: _,
+                    mode: IndirectMode::AmdgpuKernelArg,
+                } => {
+                    let i = apply(bx.cx, attrs);
+                    let byref = llvm::CreateByRefAttr(
+                        bx.cx.llcx,
+                        bx.cx.type_array(bx.cx.type_i8(), arg.layout.size.bytes()),
+                    );
+                    attributes::apply_to_callsite(
+                        callsite,
+                        llvm::AttributePlace::Argument(i),
+                        &[byref],
+                    );
+                }
                 PassMode::Direct(attrs)
-                | PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => {
+                | PassMode::Indirect {
+                    attrs,
+                    meta_attrs: None,
+                    address_space: _,
+                    mode: IndirectMode::Pointer,
+                } => {
                     apply(bx.cx, attrs);
                 }
-                PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack: _ } => {
+                PassMode::Indirect {
+                    attrs,
+                    meta_attrs: Some(meta_attrs),
+                    address_space: _,
+                    mode: _,
+                } => {
                     apply(bx.cx, attrs);
                     apply(bx.cx, meta_attrs);
                 }
diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
index d1cdf7bada0b1..63fcdf8dcbd9c 100644
--- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
@@ -2015,6 +2015,7 @@ unsafe extern "C" {
     pub(crate) fn LLVMRustCreateDereferenceableAttr(C: &Context, bytes: u64) -> &Attribute;
     pub(crate) fn LLVMRustCreateDereferenceableOrNullAttr(C: &Context, bytes: u64) -> &Attribute;
     pub(crate) fn LLVMRustCreateByValAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute;
+    pub(crate) fn LLVMRustCreateByRefAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute;
     pub(crate) fn LLVMRustCreateStructRetAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute;
     pub(crate) fn LLVMRustCreateElementTypeAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute;
     pub(crate) fn LLVMRustCreateUWTableAttr(C: &Context, async_: bool) -> &Attribute;
diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs
index 5452f4abc5c33..89e4d60656d34 100644
--- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs
@@ -122,6 +122,10 @@ pub(crate) fn CreateByValAttr<'ll>(llcx: &'ll Context, ty: &'ll Type) -> &'ll At
     unsafe { LLVMRustCreateByValAttr(llcx, ty) }
 }
 
+pub(crate) fn CreateByRefAttr<'ll>(llcx: &'ll Context, ty: &'ll Type) -> &'ll Attribute {
+    unsafe { LLVMRustCreateByRefAttr(llcx, ty) }
+}
+
 pub(crate) fn CreateStructRetAttr<'ll>(llcx: &'ll Context, ty: &'ll Type) -> &'ll Attribute {
     unsafe { LLVMRustCreateStructRetAttr(llcx, ty) }
 }
diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs
index 6b0def4ffa182..f99009a0f4243 100644
--- a/compiler/rustc_codegen_ssa/src/mir/block.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/block.rs
@@ -18,7 +18,7 @@ use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
 use rustc_middle::ty::{self, Instance, Ty, TypeVisitableExt};
 use rustc_session::config::OptLevel;
 use rustc_span::{Span, Spanned, bug, span_bug};
-use rustc_target::callconv::{ArgAbi, ArgAttributes, CastTarget, FnAbi, PassMode};
+use rustc_target::callconv::{ArgAbi, ArgAttributes, CastTarget, FnAbi, IndirectMode, PassMode};
 use tracing::{debug, info};
 
 use super::operand::OperandRef;
@@ -1257,7 +1257,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             (args, None)
         };
 
-        // Special logic for tail calls with `PassMode::Indirect { on_stack: false, .. }` arguments.
+        // Special logic for tail calls with `PassMode::Indirect { mode: IndirectMode::Pointer, .. }` arguments.
         //
         // Normally an indirect argument that is allocated in the caller's stack frame
         // would be passed as a pointer into the callee's stack frame.
@@ -1282,10 +1282,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         let mut tail_call_temporaries = vec![];
         if kind == CallKind::Tail {
             tail_call_temporaries = vec![None; first_args.len()];
-            // Copy the arguments that use `PassMode::Indirect { on_stack: false , ..}`
+            // Copy the arguments that use `PassMode::Indirect { mode: IndirectMode::Pointer , ..}`
             // to temporary stack allocations. See the comment above.
             for (i, arg) in first_args.iter().enumerate() {
-                if !matches!(fn_abi.args[i].mode, PassMode::Indirect { on_stack: false, .. }) {
+                if !matches!(
+                    fn_abi.args[i].mode,
+                    PassMode::Indirect { mode: IndirectMode::Pointer, .. }
+                ) {
                     continue;
                 }
 
@@ -1353,10 +1356,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 }
             }
 
-            let by_move = if let PassMode::Indirect { on_stack: false, .. } = fn_abi.args[i].mode
+            let by_move = if let PassMode::Indirect { mode: IndirectMode::Pointer, .. } =
+                fn_abi.args[i].mode
                 && kind == CallKind::Tail
             {
-                // Special logic for tail calls with `PassMode::Indirect { on_stack: false, .. }` arguments.
+                // Special logic for tail calls with `PassMode::Indirect { mode: IndirectMode::Pointer, .. }` arguments.
                 //
                 // Normally an indirect argument that is allocated in the caller's stack frame
                 // would be passed as a pointer into the callee's stack frame.
@@ -1977,14 +1981,16 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 }
                 _ => bug!("codegen_argument: {:?} invalid for pair argument", op),
             },
-            PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => match op.val {
-                Ref(PlaceValue { llval: a, llextra: Some(b), .. }) => {
-                    llargs.push(a);
-                    llargs.push(b);
-                    return;
+            PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
+                match op.val {
+                    Ref(PlaceValue { llval: a, llextra: Some(b), .. }) => {
+                        llargs.push(a);
+                        llargs.push(b);
+                        return;
+                    }
+                    _ => bug!("codegen_argument: {:?} invalid for unsized indirect argument", op),
                 }
-                _ => bug!("codegen_argument: {:?} invalid for unsized indirect argument", op),
-            },
+            }
             _ => {}
         }
 
@@ -2014,7 +2020,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 PassMode::Ignore | PassMode::Pair(..) => unreachable!("handled above"),
             },
             Ref(op_place_val) => match arg.mode {
-                PassMode::Indirect { attrs, on_stack, .. } => {
+                PassMode::Indirect { attrs, mode, .. } => {
+                    if mode == IndirectMode::AmdgpuKernelArg {
+                        bug!("{op:?} passed as amdgpu kernel argument with abi {arg:?}");
+                    }
                     // For `foo(packed.large_field)`, and types with <4 byte alignment on x86,
                     // alignment requirements may be higher than the type's alignment, so copy
                     // to a higher-aligned alloca.
@@ -2023,7 +2032,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         None => arg.layout.align.abi,
                     };
                     // Copy to an alloca when the argument is neither by-val nor by-move.
-                    if op_place_val.align < required_align || (!on_stack && !by_move) {
+                    if op_place_val.align < required_align
+                        || (mode == IndirectMode::Pointer && !by_move)
+                    {
                         let scratch = PlaceValue::alloca(bx, arg.layout.size, required_align);
                         bx.lifetime_start(scratch.llval, arg.layout.size);
                         op.store_with_annotation(bx, scratch.with_type(arg.layout));
@@ -2036,8 +2047,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 _ => (op_place_val.llval, op_place_val.align, true),
             },
             ZeroSized => match arg.mode {
-                PassMode::Indirect { on_stack, .. } => {
-                    if on_stack {
+                PassMode::Indirect { mode, .. } => {
+                    if mode == IndirectMode::AmdgpuKernelArg {
+                        bug!("{op:?} passed as amdgpu kernel argument with abi {arg:?}");
+                    }
+                    if mode == IndirectMode::OnStack {
                         // It doesn't seem like any target can have `byval` ZSTs, so this assert
                         // is here to replace a would-be untested codepath.
                         bug!("ZST {op:?} passed on stack with abi {arg:?}");
diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs
index b5cecf4b5c434..aefa8356536dc 100644
--- a/compiler/rustc_codegen_ssa/src/mir/mod.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs
@@ -8,7 +8,7 @@ use rustc_middle::mir::{Body, Local, UnwindTerminateReason, traversal};
 use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, HasTypingEnv, TyAndLayout};
 use rustc_middle::ty::{self, Instance, Ty, TyCtxt, TypeFoldable, TypeVisitableExt};
 use rustc_span::{ErrorGuaranteed, bug, span_bug};
-use rustc_target::callconv::{FnAbi, PassMode};
+use rustc_target::callconv::{FnAbi, IndirectMode, PassMode};
 use tracing::{debug, instrument};
 
 use crate::base;
@@ -561,15 +561,21 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 
             match arg.mode {
                 // Sized indirect arguments
-                PassMode::Indirect { attrs, meta_attrs: None, on_stack: _ } => {
+                PassMode::Indirect { attrs, meta_attrs: None, address_space: _, mode } => {
                     // Don't copy an indirect argument to an alloca, the caller already put it
                     // in a temporary alloca and gave it up.
+                    // AmdgpuKernelArg/byref arguments must not be modified, so always create a
+                    // local alloca for them.
+                    // If the argument is underaligned, then we need to copy it to a higher-aligned
+                    // alloca.
                     // FIXME: lifetimes
+                    let mut needs_alloca = mode == IndirectMode::AmdgpuKernelArg;
                     if let Some(pointee_align) = attrs.pointee_align
                         && pointee_align < arg.layout.align.abi
                     {
-                        // ...unless the argument is underaligned, then we need to copy it to
-                        // a higher-aligned alloca.
+                        needs_alloca = true;
+                    }
+                    if needs_alloca {
                         let tmp = PlaceRef::alloca(bx, arg.layout);
                         bx.store_fn_arg(arg, &mut llarg_idx, tmp);
                         LocalRef::Place(tmp)
@@ -580,7 +586,7 @@ fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
                     }
                 }
                 // Unsized indirect arguments
-                PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
+                PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ } => {
                     // As the storage for the indirect argument lives during
                     // the whole function call, we just copy the wide pointer.
                     let llarg = bx.get_param(llarg_idx);
diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
index 161b5bdb952d3..bc8fa60b66a52 100644
--- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
+++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
@@ -480,6 +480,11 @@ extern "C" LLVMAttributeRef LLVMRustCreateByValAttr(LLVMContextRef C,
   return wrap(Attribute::getWithByValType(*unwrap(C), unwrap(Ty)));
 }
 
+extern "C" LLVMAttributeRef LLVMRustCreateByRefAttr(LLVMContextRef C,
+                                                    LLVMTypeRef Ty) {
+  return wrap(Attribute::getWithByRefType(*unwrap(C), unwrap(Ty)));
+}
+
 extern "C" LLVMAttributeRef LLVMRustCreateStructRetAttr(LLVMContextRef C,
                                                         LLVMTypeRef Ty) {
   return wrap(Attribute::getWithStructRetType(*unwrap(C), unwrap(Ty)));
diff --git a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs
index 5bba125aefc58..8814670ca4300 100644
--- a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs
+++ b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs
@@ -135,7 +135,7 @@ impl<'tcx> Visitor<'tcx> for DeduceParamAttrs {
             }
 
             // Like a call, but more conservative because the backend may introduce writes to an
-            // argument if the argument is passed as `PassMode::Indirect { on_stack: false, ... }`.
+            // argument if the argument is passed as `PassMode::Indirect { mode: IndirectMode::Pointer, ... }`.
             TerminatorKind::TailCall { .. } => {
                 for usage in self.usage.iter_mut() {
                     *usage |= UsageSummary::MUTATE;
diff --git a/compiler/rustc_public/src/abi.rs b/compiler/rustc_public/src/abi.rs
index b760ed98c7111..c6d1d2c77a13f 100644
--- a/compiler/rustc_public/src/abi.rs
+++ b/compiler/rustc_public/src/abi.rs
@@ -41,6 +41,19 @@ pub struct ArgAbi {
     pub mode: PassMode,
 }
 
+/// Different modes in which indirect arguments can be passed.
+#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, Serialize)]
+pub enum IndirectMode {
+    /// Passed as a normal pointer, nothing special.
+    Pointer,
+    /// The value is placed at a fixed stack offset rather than passed as a regular pointer
+    /// argument.
+    OnStack,
+    /// Similar to `OnStack` except that the pointer does not necessarily point to the stack, no
+    /// extra copy is made, and the passed argument should not be modified.
+    AmdgpuKernelArg,
+}
+
 /// How a function argument should be passed in to the target function.
 ///
 /// The pass mode is determined by the platform's calling convention and the
@@ -74,14 +87,13 @@ pub enum PassMode {
     /// Pass the argument indirectly via a pointer.
     ///
     /// The caller places the value in memory and passes a pointer to it.
-    /// When `on_stack` is true, the value is placed at a fixed stack offset
-    /// rather than passed as a regular pointer argument.
     Indirect {
         attrs: ArgAttributes,
         /// Attributes for the metadata pointer (vtable or length) of unsized arguments.
         /// Only present for unsized types (e.g., `dyn Trait`, `[T]`).
         meta_attrs: Option<ArgAttributes>,
-        on_stack: bool,
+        address_space: Option<AddressSpace>,
+        mode: IndirectMode,
     },
 }
 
diff --git a/compiler/rustc_public/src/unstable/convert/stable/abi.rs b/compiler/rustc_public/src/unstable/convert/stable/abi.rs
index 766c522958db7..65b8e9bd72761 100644
--- a/compiler/rustc_public/src/unstable/convert/stable/abi.rs
+++ b/compiler/rustc_public/src/unstable/convert/stable/abi.rs
@@ -11,9 +11,9 @@ use rustc_target::callconv;
 use crate::IndexedVal;
 use crate::abi::{
     AddressSpace, ArgAbi, ArgAttributes, ArgExtension, CallConvention, CastTarget, FieldsShape,
-    FloatLength, FnAbi, IntegerLength, IntegerType, Layout, LayoutShape, NumScalableVectors,
-    PassMode, Primitive, Reg, RegKind, ReprFlags, ReprOptions, Scalar, TagEncoding, TyAndLayout,
-    Uniform, ValueRepr, VariantFields, VariantsShape, WrappingRange,
+    FloatLength, FnAbi, IndirectMode, IntegerLength, IntegerType, Layout, LayoutShape,
+    NumScalableVectors, PassMode, Primitive, Reg, RegKind, ReprFlags, ReprOptions, Scalar,
+    TagEncoding, TyAndLayout, Uniform, ValueRepr, VariantFields, VariantsShape, WrappingRange,
 };
 use crate::compiler_interface::BridgeTys;
 use crate::target::MachineSize as Size;
@@ -155,6 +155,22 @@ impl<'tcx> Stable<'tcx> for CanonAbi {
     }
 }
 
+impl<'tcx> Stable<'tcx> for callconv::IndirectMode {
+    type T = IndirectMode;
+
+    fn stable<'cx>(
+        &self,
+        _tables: &mut Tables<'cx, BridgeTys>,
+        _cx: &CompilerCtxt<'cx, BridgeTys>,
+    ) -> Self::T {
+        match self {
+            callconv::IndirectMode::Pointer => IndirectMode::Pointer,
+            callconv::IndirectMode::OnStack => IndirectMode::OnStack,
+            callconv::IndirectMode::AmdgpuKernelArg => IndirectMode::AmdgpuKernelArg,
+        }
+    }
+}
+
 impl<'tcx> Stable<'tcx> for callconv::PassMode {
     type T = PassMode;
 
@@ -172,11 +188,14 @@ impl<'tcx> Stable<'tcx> for callconv::PassMode {
             callconv::PassMode::Cast { pad_i32_count, cast } => {
                 PassMode::Cast { pad_i32_count: *pad_i32_count, cast: cast.stable(tables, cx) }
             }
-            callconv::PassMode::Indirect { attrs, meta_attrs, on_stack } => PassMode::Indirect {
-                attrs: attrs.stable(tables, cx),
-                meta_attrs: meta_attrs.map(|a| a.stable(tables, cx)),
-                on_stack: *on_stack,
-            },
+            callconv::PassMode::Indirect { attrs, meta_attrs, address_space, mode } => {
+                PassMode::Indirect {
+                    attrs: attrs.stable(tables, cx),
+                    meta_attrs: meta_attrs.map(|a| a.stable(tables, cx)),
+                    address_space: address_space.stable(tables, cx),
+                    mode: mode.stable(tables, cx),
+                }
+            }
         }
     }
 }
diff --git a/compiler/rustc_target/src/callconv/mod.rs b/compiler/rustc_target/src/callconv/mod.rs
index 9fe22a3a174b6..474f45b54e9b2 100644
--- a/compiler/rustc_target/src/callconv/mod.rs
+++ b/compiler/rustc_target/src/callconv/mod.rs
@@ -36,6 +36,25 @@ mod x86_win32;
 mod x86_win64;
 mod xtensa;
 
+/// Different modes in which indirect arguments can be passed.
+#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, StableHash)]
+pub enum IndirectMode {
+    /// Passed as a normal pointer, nothing special.
+    Pointer,
+    /// The value should be passed at a fixed stack offset in accordance to
+    /// the ABI rather than passed using a pointer. This corresponds to the `byval` LLVM argument
+    /// attribute. The `byval` argument will use a byte array with the same size as the Rust type
+    /// (which ensures that padding is preserved and that we do not rely on LLVM's struct layout),
+    /// and will use the alignment specified in `attrs.pointee_align` (if `Some`) or the type's
+    /// alignment (if `None`). This means that the alignment will not always
+    /// match the Rust type's alignment; see documentation of `pass_by_stack_offset` for more info.
+    OnStack,
+    /// `AmdgpuKernelArg` behaves similar to `OnStack` except that the pointer does not necessarily
+    /// point to the stack, no extra copy is made, and the passed argument should not be modified.
+    /// This corresponds to the `byref` LLVM argument attribute.
+    AmdgpuKernelArg,
+}
+
 #[derive(Clone, PartialEq, Eq, Hash, Debug, StableHash)]
 pub enum PassMode {
     /// Ignore the argument.
@@ -63,16 +82,17 @@ pub enum PassMode {
     /// The `meta_attrs` value, if any, is for the metadata (vtable or length) of an unsized
     /// argument. (This is the only mode that supports unsized arguments.)
     ///
-    /// `on_stack` defines that the value should be passed at a fixed stack offset in accordance to
-    /// the ABI rather than passed using a pointer. This corresponds to the `byval` LLVM argument
-    /// attribute. The `byval` argument will use a byte array with the same size as the Rust type
-    /// (which ensures that padding is preserved and that we do not rely on LLVM's struct layout),
-    /// and will use the alignment specified in `attrs.pointee_align` (if `Some`) or the type's
-    /// alignment (if `None`). This means that the alignment will not always
-    /// match the Rust type's alignment; see documentation of `pass_by_stack_offset` for more info.
+    /// `address_space` specifies if the pointer is in a special address space or the default one.
     ///
-    /// `on_stack` cannot be true for unsized arguments, i.e., when `meta_attrs` is `Some`.
-    Indirect { attrs: ArgAttributes, meta_attrs: Option<ArgAttributes>, on_stack: bool },
+    /// `mode` can be a special way to pass an argument indirectly.
+    /// `OnStack` and `AmdgpuKernelArg` cannot be used for unsized arguments, i.e., when
+    /// `meta_attrs` is `Some`.
+    Indirect {
+        attrs: ArgAttributes,
+        meta_attrs: Option<ArgAttributes>,
+        address_space: Option<AddressSpace>,
+        mode: IndirectMode,
+    },
 }
 
 impl PassMode {
@@ -89,13 +109,23 @@ impl PassMode {
                 PassMode::Cast { cast: c2, pad_i32_count: pad2 },
             ) => c1.eq_abi(c2) && pad1 == pad2,
             (
-                PassMode::Indirect { attrs: a1, meta_attrs: None, on_stack: s1 },
-                PassMode::Indirect { attrs: a2, meta_attrs: None, on_stack: s2 },
-            ) => a1.eq_abi(a2) && s1 == s2,
+                PassMode::Indirect { attrs: a1, meta_attrs: None, address_space: as1, mode: m1 },
+                PassMode::Indirect { attrs: a2, meta_attrs: None, address_space: as2, mode: m2 },
+            ) => a1.eq_abi(a2) && as1 == as2 && m1 == m2,
             (
-                PassMode::Indirect { attrs: a1, meta_attrs: Some(e1), on_stack: s1 },
-                PassMode::Indirect { attrs: a2, meta_attrs: Some(e2), on_stack: s2 },
-            ) => a1.eq_abi(a2) && e1.eq_abi(e2) && s1 == s2,
+                PassMode::Indirect {
+                    attrs: a1,
+                    meta_attrs: Some(e1),
+                    address_space: as1,
+                    mode: m1,
+                },
+                PassMode::Indirect {
+                    attrs: a2,
+                    meta_attrs: Some(e2),
+                    address_space: as2,
+                    mode: m2,
+                },
+            ) => a1.eq_abi(a2) && as1 == as2 && e1.eq_abi(e2) && m1 == m2,
             _ => false,
         }
     }
@@ -424,7 +454,7 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
 
         let meta_attrs = layout.is_unsized().then_some(ArgAttributes::new());
 
-        PassMode::Indirect { attrs, meta_attrs, on_stack: false }
+        PassMode::Indirect { attrs, meta_attrs, address_space: None, mode: IndirectMode::Pointer }
     }
 
     /// Pass this argument indirectly, by passing a (thin or wide) pointer to the argument instead.
@@ -435,13 +465,31 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
             PassMode::Direct(_) | PassMode::Pair(_, _) => {
                 self.mode = Self::indirect_pass_mode(&self.layout);
             }
-            PassMode::Indirect { attrs: _, meta_attrs: _, on_stack: false } => {
+            PassMode::Indirect {
+                attrs: _,
+                meta_attrs: _,
+                address_space: _,
+                mode: IndirectMode::Pointer,
+            } => {
                 // already indirect
             }
             _ => panic!("Tried to make {:?} indirect", self.mode),
         }
     }
 
+    /// Pass this argument indirectly, by passing a (thin or wide) pointer to the argument instead.
+    /// This is valid for both sized and unsized arguments.
+    #[track_caller]
+    pub fn make_indirect_addrspace(&mut self, addrspace: AddressSpace) {
+        self.make_indirect();
+        match self.mode {
+            PassMode::Indirect { ref mut address_space, .. } => {
+                *address_space = Some(addrspace);
+            }
+            _ => unreachable!(),
+        }
+    }
+
     /// Same as `make_indirect`, but for arguments that are ignored. Only needed for ABIs that pass
     /// ZSTs indirectly.
     #[track_caller]
@@ -450,7 +498,12 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
             PassMode::Ignore => {
                 self.mode = Self::indirect_pass_mode(&self.layout);
             }
-            PassMode::Indirect { attrs: _, meta_attrs: _, on_stack: false } => {
+            PassMode::Indirect {
+                attrs: _,
+                meta_attrs: _,
+                address_space: _,
+                mode: IndirectMode::Pointer,
+            } => {
                 // already indirect
             }
             _ => panic!("Tried to make {:?} indirect (expected `PassMode::Ignore`)", self.mode),
@@ -477,8 +530,8 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
         assert!(!self.layout.is_unsized(), "used byval ABI for unsized layout");
         self.make_indirect();
         match self.mode {
-            PassMode::Indirect { ref mut attrs, meta_attrs: _, ref mut on_stack } => {
-                *on_stack = true;
+            PassMode::Indirect { ref mut attrs, meta_attrs: _, address_space: _, ref mut mode } => {
+                *mode = IndirectMode::OnStack;
 
                 // Some platforms, like 32-bit x86, change the alignment of the type when passing
                 // `byval`. Account for that.
@@ -492,6 +545,22 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
         }
     }
 
+    /// Pass this argument indirectly.
+    /// This corresponds to the `byref` LLVM argument attribute.
+    ///
+    /// `address_space` specifies the address space of the passed pointer.
+    pub fn pass_amdgpu_kernel_arg(&mut self, addrspace: Option<AddressSpace>) {
+        assert!(!self.layout.is_unsized(), "used amdgpu kernel arg ABI for unsized layout");
+        self.make_indirect();
+        match self.mode {
+            PassMode::Indirect { attrs: _, meta_attrs: _, ref mut address_space, ref mut mode } => {
+                *mode = IndirectMode::AmdgpuKernelArg;
+                *address_space = addrspace;
+            }
+            _ => unreachable!(),
+        }
+    }
+
     pub fn extend_integer_width_to(&mut self, bits: u64) {
         // Only integers have signedness
         if let BackendRepr::Scalar(scalar) = self.layout.backend_repr
@@ -545,11 +614,17 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
     }
 
     pub fn is_sized_indirect(&self) -> bool {
-        matches!(self.mode, PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ })
+        matches!(
+            self.mode,
+            PassMode::Indirect { attrs: _, meta_attrs: None, address_space: _, mode: _ }
+        )
     }
 
     pub fn is_unsized_indirect(&self) -> bool {
-        matches!(self.mode, PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ })
+        matches!(
+            self.mode,
+            PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ }
+        )
     }
 
     pub fn is_ignore(&self) -> bool {
@@ -834,7 +909,7 @@ impl<'a, Ty> FnAbi<'a, Ty> {
                     // Compute `Aggregate` ABI.
 
                     let is_indirect_not_on_stack =
-                        matches!(arg.mode, PassMode::Indirect { on_stack: false, .. });
+                        matches!(arg.mode, PassMode::Indirect { mode: IndirectMode::Pointer, .. });
                     assert!(is_indirect_not_on_stack);
 
                     let size = arg.layout.size;
@@ -949,7 +1024,7 @@ mod size_asserts {
 
     use super::*;
     // tidy-alphabetical-start
-    static_assert_size!(ArgAbi<'_, usize>, 56);
-    static_assert_size!(FnAbi<'_, usize>, 80);
+    static_assert_size!(ArgAbi<'_, usize>, 64);
+    static_assert_size!(FnAbi<'_, usize>, 88);
     // tidy-alphabetical-end
 }
diff --git a/compiler/rustc_target/src/callconv/x86.rs b/compiler/rustc_target/src/callconv/x86.rs
index fd608fcf62919..f51e29b34e1d3 100644
--- a/compiler/rustc_target/src/callconv/x86.rs
+++ b/compiler/rustc_target/src/callconv/x86.rs
@@ -167,12 +167,13 @@ pub(crate) fn fill_inregs<'a, Ty, C>(
 
     for arg in fn_abi.args.iter_mut() {
         let attrs = match arg.mode {
-            PassMode::Ignore | PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => {
+            PassMode::Ignore
+            | PassMode::Indirect { attrs: _, meta_attrs: None, address_space: _, mode: _ } => {
                 continue;
             }
             PassMode::Direct(ref mut attrs) => attrs,
             PassMode::Pair(..)
-            | PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ }
+            | PassMode::Indirect { attrs: _, meta_attrs: Some(_), address_space: _, mode: _ }
             | PassMode::Cast { .. } => {
                 unreachable!("x86 shouldn't be passing arguments by {:?}", arg.mode)
             }
diff --git a/compiler/rustc_target/src/callconv/xtensa.rs b/compiler/rustc_target/src/callconv/xtensa.rs
index 4dc9fad650636..49005adeb33c0 100644
--- a/compiler/rustc_target/src/callconv/xtensa.rs
+++ b/compiler/rustc_target/src/callconv/xtensa.rs
@@ -7,7 +7,7 @@
 
 use rustc_abi::{BackendRepr, HasDataLayout, Size, TyAbiInterface};
 
-use crate::callconv::{ArgAbi, FnAbi, Reg, Uniform};
+use crate::callconv::{ArgAbi, FnAbi, IndirectMode, Reg, Uniform};
 use crate::spec::HasTargetSpec;
 
 const NUM_ARG_GPRS: u64 = 6;
@@ -29,8 +29,8 @@ where
     classify_arg_ty(cx, arg, &mut arg_gprs_left, true);
     // Ret args cannot be passed via stack, we lower to indirect and let the backend handle the invisible reference
     match arg.mode {
-        super::PassMode::Indirect { attrs: _, meta_attrs: _, ref mut on_stack } => {
-            *on_stack = false;
+        super::PassMode::Indirect { attrs: _, meta_attrs: _, address_space: _, ref mut mode } => {
+            *mode = IndirectMode::Pointer;
         }
         _ => {}
     }
diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs
index 55140d2c5458d..e8f9ded9562d5 100644
--- a/compiler/rustc_ty_utils/src/abi.rs
+++ b/compiler/rustc_ty_utils/src/abi.rs
@@ -12,7 +12,9 @@ use rustc_middle::ty::layout::{
 use rustc_middle::ty::{self, InstanceKind, ShimKind, Ty, TyCtxt, Unnormalized};
 use rustc_span::def_id::DefId;
 use rustc_span::{DUMMY_SP, bug};
-use rustc_target::callconv::{AbiMap, ArgAbi, ArgAttribute, ArgAttributes, FnAbi, PassMode};
+use rustc_target::callconv::{
+    AbiMap, ArgAbi, ArgAttribute, ArgAttributes, FnAbi, IndirectMode, PassMode,
+};
 use tracing::debug;
 
 pub(crate) fn provide(providers: &mut Providers) {
@@ -444,15 +446,15 @@ fn fn_abi_sanity_check<'tcx>(
                 // omitted entirely in the calling convention.
                 assert!(arg.is_ignore());
             }
-            if let PassMode::Indirect { on_stack, .. } = arg.mode
+            if let PassMode::Indirect { mode, .. } = arg.mode
                 && spec_abi != ExternAbi::RustTail
             {
-                assert!(!on_stack, "rustic abi {spec_abi:?} shouldn't use on_stack");
+                assert!(mode == IndirectMode::Pointer, "rust abi must use plain pointer mode");
             }
         } else if arg.layout.pass_indirectly_in_non_rustic_abis(cx) {
             assert_matches!(
                 arg.mode,
-                PassMode::Indirect { on_stack: false, .. },
+                PassMode::Indirect { mode: IndirectMode::Pointer, .. },
                 "the {spec_abi} ABI does not implement `#[rustc_pass_indirectly_in_non_rustic_abis]`"
             );
         }
@@ -506,9 +508,9 @@ fn fn_abi_sanity_check<'tcx>(
                 // Indirect returns are arguments from an ABI perspective.
                 fn_arg_attrs_sanity_check(attrs, false);
             }
-            PassMode::Indirect { meta_attrs: Some(meta_attrs), attrs, on_stack } => {
+            PassMode::Indirect { meta_attrs: Some(meta_attrs), attrs, address_space: _, mode } => {
                 // With metadata. Must be unsized and not on the stack.
-                assert!(arg.layout.is_unsized() && !on_stack);
+                assert!(arg.layout.is_unsized() && *mode == IndirectMode::Pointer);
                 // Also, must not be `extern` type.
                 let tail = tcx.struct_tail_for_codegen(arg.layout.ty, cx.typing_env);
                 if matches!(tail.kind(), ty::Foreign(..)) {
diff --git a/tests/assembly-llvm/tail-call-indirect.rs b/tests/assembly-llvm/tail-call-indirect.rs
index 2bc1743a9bafd..918283966b405 100644
--- a/tests/assembly-llvm/tail-call-indirect.rs
+++ b/tests/assembly-llvm/tail-call-indirect.rs
@@ -10,10 +10,10 @@
 #![no_core]
 #![crate_type = "lib"]
 
-// Test tail calls with `PassMode::Indirect { on_stack: false, .. }` arguments.
+// Test tail calls with `PassMode::Indirect { mode: IndirectMode::Pointer, .. }` arguments.
 //
-// Normally an indirect argument with `on_stack: false` would be passed as a pointer to the
-// caller's stack frame. For tail calls, that would be unsound, because the caller's stack
+// Normally an indirect argument with `mode: IndirectMode::Pointer` would be passed as a pointer to
+// the caller's stack frame. For tail calls, that would be unsound, because the caller's stack
 // frame is overwritten by the callee's stack frame.
 //
 // The solution is to write the argument into the caller's argument place (stored somewhere further
diff --git a/tests/ui-fulldeps/rustc_public/check_abi.rs b/tests/ui-fulldeps/rustc_public/check_abi.rs
index f6c95fb745409..92312cd4c8712 100644
--- a/tests/ui-fulldeps/rustc_public/check_abi.rs
+++ b/tests/ui-fulldeps/rustc_public/check_abi.rs
@@ -15,8 +15,8 @@ extern crate rustc_middle;
 extern crate rustc_public;
 
 use rustc_public::abi::{
-    ArgAbi, ArgExtension, CallConvention, FieldsShape, IntegerLength, PassMode, Primitive, Scalar,
-    ValueRepr, VariantsShape,
+    ArgAbi, ArgExtension, CallConvention, FieldsShape, IndirectMode, IntegerLength, PassMode,
+    Primitive, Scalar, ValueRepr, VariantsShape,
 };
 use rustc_public::mir::MirVisitor;
 use rustc_public::mir::mono::Instance;
@@ -122,14 +122,14 @@ fn check_primitive(abi: &ArgAbi) {
 /// Check the return value: `Result<usize, &str>`.
 fn check_result(abi: &ArgAbi) {
     assert!(abi.ty.kind().is_enum());
-    let PassMode::Indirect { ref attrs, ref meta_attrs, on_stack } = abi.mode else {
+    let PassMode::Indirect { ref attrs, ref meta_attrs, address_space: _, mode } = abi.mode else {
         panic!("Expected PassMode::Indirect for Result, got: {:?}", abi.mode);
     };
     // Indirect arguments have a pointee alignment (the pointer must be aligned).
     assert!(attrs.pointee_align().is_some());
     // Result is a sized type, so no metadata pointer.
     assert!(meta_attrs.is_none());
-    assert!(!on_stack);
+    assert!(mode == IndirectMode::Pointer);
     let layout = abi.layout.shape();
     assert!(layout.is_sized());
     assert_matches!(layout.fields, FieldsShape::Arbitrary { .. });
diff --git a/tests/ui-fulldeps/rustc_public/check_abi_cast.rs b/tests/ui-fulldeps/rustc_public/check_abi_cast.rs
index 0bd4ac684066e..a54abdd5deeaf 100644
--- a/tests/ui-fulldeps/rustc_public/check_abi_cast.rs
+++ b/tests/ui-fulldeps/rustc_public/check_abi_cast.rs
@@ -23,7 +23,7 @@ use std::convert::TryFrom;
 use std::io::Write;
 use std::ops::ControlFlow;
 
-use rustc_public::abi::{CallConvention, PassMode, RegKind};
+use rustc_public::abi::{CallConvention, IndirectMode, PassMode, RegKind};
 use rustc_public::mir::mono::Instance;
 use rustc_public::{CrateDef, ItemKind};
 
@@ -147,7 +147,7 @@ fn test_abi_cast() -> ControlFlow<()> {
     }
     // Fourth TwoWords has no registers left → Indirect (on stack)
     assert!(
-        matches!(&abi.args[3].mode, PassMode::Indirect { on_stack: true, .. }),
+        matches!(&abi.args[3].mode, PassMode::Indirect { mode: IndirectMode::OnStack, .. }),
         "Expected arg 3 to be Indirect on stack, got: {:?}",
         abi.args[3].mode
     );
diff --git a/tests/ui/abi/c-zst.powerpc-linux.stderr b/tests/ui/abi/c-zst.powerpc-linux.stderr
index edea2d5772280..cefc2ba085182 100644
--- a/tests/ui/abi/c-zst.powerpc-linux.stderr
+++ b/tests/ui/abi/c-zst.powerpc-linux.stderr
@@ -35,7 +35,8 @@ error: fn_abi_of(pass_zst) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/abi/c-zst.s390x-linux.stderr b/tests/ui/abi/c-zst.s390x-linux.stderr
index edea2d5772280..cefc2ba085182 100644
--- a/tests/ui/abi/c-zst.s390x-linux.stderr
+++ b/tests/ui/abi/c-zst.s390x-linux.stderr
@@ -35,7 +35,8 @@ error: fn_abi_of(pass_zst) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/abi/c-zst.sparc64-linux.stderr b/tests/ui/abi/c-zst.sparc64-linux.stderr
index edea2d5772280..cefc2ba085182 100644
--- a/tests/ui/abi/c-zst.sparc64-linux.stderr
+++ b/tests/ui/abi/c-zst.sparc64-linux.stderr
@@ -35,7 +35,8 @@ error: fn_abi_of(pass_zst) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr b/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr
index edea2d5772280..cefc2ba085182 100644
--- a/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr
+++ b/tests/ui/abi/c-zst.x86_64-pc-windows-gnu.stderr
@@ -35,7 +35,8 @@ error: fn_abi_of(pass_zst) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/abi/debug.generic.stderr b/tests/ui/abi/debug.generic.stderr
index 6242d93b09534..1793674fa462a 100644
--- a/tests/ui/abi/debug.generic.stderr
+++ b/tests/ui/abi/debug.generic.stderr
@@ -446,7 +446,8 @@ error: ABIs are not compatible
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
@@ -519,7 +520,8 @@ error: ABIs are not compatible
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/abi/debug.loongarch64.stderr b/tests/ui/abi/debug.loongarch64.stderr
index 176c68ecd4c7b..29ec7846101f1 100644
--- a/tests/ui/abi/debug.loongarch64.stderr
+++ b/tests/ui/abi/debug.loongarch64.stderr
@@ -446,7 +446,8 @@ error: ABIs are not compatible
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
@@ -519,7 +520,8 @@ error: ABIs are not compatible
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/abi/debug.riscv64.stderr b/tests/ui/abi/debug.riscv64.stderr
index 176c68ecd4c7b..29ec7846101f1 100644
--- a/tests/ui/abi/debug.riscv64.stderr
+++ b/tests/ui/abi/debug.riscv64.stderr
@@ -446,7 +446,8 @@ error: ABIs are not compatible
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
@@ -519,7 +520,8 @@ error: ABIs are not compatible
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/abi/pass-indirectly-attr.rs b/tests/ui/abi/pass-indirectly-attr.rs
index 54aafc716587c..bb90b8354ea91 100644
--- a/tests/ui/abi/pass-indirectly-attr.rs
+++ b/tests/ui/abi/pass-indirectly-attr.rs
@@ -20,7 +20,7 @@ pub struct Type(u8);
 pub extern "C" fn extern_c(_: Type) {}
 //~^ ERROR fn_abi_of(extern_c) = FnAbi {
 //~| ERROR mode: Indirect
-//~| ERROR on_stack: false,
+//~| ERROR mode: Pointer,
 //~| ERROR conv: C,
 
 #[rustc_abi(debug)]
diff --git a/tests/ui/abi/pass-indirectly-attr.stderr b/tests/ui/abi/pass-indirectly-attr.stderr
index efeec0d86982b..5821e6279bb85 100644
--- a/tests/ui/abi/pass-indirectly-attr.stderr
+++ b/tests/ui/abi/pass-indirectly-attr.stderr
@@ -48,7 +48,8 @@ error: fn_abi_of(extern_c) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/c-variadic/pass-by-value-abi.aarch64.stderr b/tests/ui/c-variadic/pass-by-value-abi.aarch64.stderr
index 45edd7bc0e0ee..c9e77ac941901 100644
--- a/tests/ui/c-variadic/pass-by-value-abi.aarch64.stderr
+++ b/tests/ui/c-variadic/pass-by-value-abi.aarch64.stderr
@@ -35,7 +35,8 @@ error: fn_abi_of(take_va_list) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/c-variadic/pass-by-value-abi.rs b/tests/ui/c-variadic/pass-by-value-abi.rs
index bcca09e90438a..317840601c050 100644
--- a/tests/ui/c-variadic/pass-by-value-abi.rs
+++ b/tests/ui/c-variadic/pass-by-value-abi.rs
@@ -27,9 +27,9 @@ use std::ffi::VaList;
 pub extern "C" fn take_va_list(_: VaList<'_>) {}
 //~^ ERROR fn_abi_of(take_va_list) = FnAbi {
 //[x86_64]~^^ ERROR mode: Indirect {
-//[x86_64]~^^^ ERROR on_stack: false,
+//[x86_64]~^^^ ERROR mode: Pointer,
 //[aarch64]~^^^^ ERROR mode: Indirect {
-//[aarch64]~^^^^^ ERROR on_stack: false,
+//[aarch64]~^^^^^ ERROR mode: Pointer,
 //[win]~^^^^^^ ERROR mode: Direct(
 
 #[cfg(all(target_arch = "x86_64", not(windows)))]
@@ -37,11 +37,11 @@ pub extern "C" fn take_va_list(_: VaList<'_>) {}
 pub extern "sysv64" fn take_va_list_sysv64(_: VaList<'_>) {}
 //[x86_64]~^ ERROR fn_abi_of(take_va_list_sysv64) = FnAbi {
 //[x86_64]~^^ ERROR mode: Indirect {
-//[x86_64]~^^^ ERROR on_stack: false,
+//[x86_64]~^^^ ERROR mode: Pointer,
 
 #[cfg(all(target_arch = "x86_64", not(windows)))]
 #[rustc_abi(debug)]
 pub extern "win64" fn take_va_list_win64(_: VaList<'_>) {}
 //[x86_64]~^ ERROR: fn_abi_of(take_va_list_win64) = FnAbi {
 //[x86_64]~^^ ERROR mode: Indirect {
-//[x86_64]~^^^ ERROR on_stack: false,
+//[x86_64]~^^^ ERROR mode: Pointer,
diff --git a/tests/ui/c-variadic/pass-by-value-abi.x86_64.stderr b/tests/ui/c-variadic/pass-by-value-abi.x86_64.stderr
index 1e203b93e66b3..04320a5312361 100644
--- a/tests/ui/c-variadic/pass-by-value-abi.x86_64.stderr
+++ b/tests/ui/c-variadic/pass-by-value-abi.x86_64.stderr
@@ -35,7 +35,8 @@ error: fn_abi_of(take_va_list) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
@@ -113,7 +114,8 @@ error: fn_abi_of(take_va_list_sysv64) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
@@ -193,7 +195,8 @@ error: fn_abi_of(take_va_list_win64) = FnAbi {
                            ),
                        },
                        meta_attrs: None,
-                       on_stack: false,
+                       address_space: None,
+                       mode: Pointer,
                    },
                },
            ],
diff --git a/tests/ui/explicit-tail-calls/indirect.rs b/tests/ui/explicit-tail-calls/indirect.rs
index b3e2613efad25..71107ef420c35 100644
--- a/tests/ui/explicit-tail-calls/indirect.rs
+++ b/tests/ui/explicit-tail-calls/indirect.rs
@@ -25,17 +25,17 @@
 #![feature(explicit_tail_calls)]
 #![expect(incomplete_features)]
 
-// Test tail calls with `PassMode::Indirect { on_stack: false, .. }` arguments.
+// Test tail calls with `PassMode::Indirect { mode: IndirectMode::Pointer, .. }` arguments.
 //
-// Normally an indirect argument with `on_stack: false` would be passed as a pointer to the
-// caller's stack frame. For tail calls, that would be unsound, because the caller's stack
+// Normally an indirect argument with `mode: IndirectMode::Pointer` would be passed as a pointer to
+// the caller's stack frame. For tail calls, that would be unsound, because the caller's stack
 // frame is overwritten by the callee's stack frame.
 //
 // The solution is to write the argument into the caller's argument place (stored somewhere further
 // up the stack), and forward that place.
 
 // A struct big enough that it is not passed via registers, so that the rust calling convention uses
-// `Indirect { on_stack: false, .. }`.
+// `Indirect { mode: IndirectMode::Pointer, .. }`.
 #[repr(C)]
 #[derive(Default, Debug, Clone, Copy, PartialEq, Eq)]
 pub struct Big([u64; 4]);
@@ -79,7 +79,7 @@ fn main() {
     assert_eq!(update_in_caller(Big::default()), 0 + 2 + 3 + 4);
 
     assert_eq!(swapper(u8::MIN, u8::MAX), (u8::MAX, u8::MIN));
-    // i128 uses `PassMode::Indirect { on_stack: false, .. }` on x86_64 MSVC.
+    // i128 uses `PassMode::Indirect { mode: IndirectMode::Pointer, .. }` on x86_64 MSVC.
     assert_eq!(swapper(i128::MIN, i128::MAX), (i128::MAX, i128::MIN));
     assert_eq!(swapper(Big([1; 4]), Big([2; 4])), (Big([2; 4]), Big([1; 4])));
 

From bf91751cda156af266c0e731089783970c8de368 Mon Sep 17 00:00:00 2001
From: Flakebi <flakebi@t-online.de>
Date: Thu, 3 Sep 2026 09:21:39 +0200
Subject: [PATCH 04/28] Pre-commit amdgpu gpu-kernel ABI test

---
 tests/codegen-llvm/amdgpu-abi/struct-abi.rs | 133 ++++++++++++++++++++
 1 file changed, 133 insertions(+)
 create mode 100644 tests/codegen-llvm/amdgpu-abi/struct-abi.rs

diff --git a/tests/codegen-llvm/amdgpu-abi/struct-abi.rs b/tests/codegen-llvm/amdgpu-abi/struct-abi.rs
new file mode 100644
index 0000000000000..bf51cbfa7f7a0
--- /dev/null
+++ b/tests/codegen-llvm/amdgpu-abi/struct-abi.rs
@@ -0,0 +1,133 @@
+//@ add-minicore
+//@ compile-flags: --crate-type=rlib --target=amdgcn-amd-amdhsa -Ctarget-cpu=gfx900 -Copt-level=3
+//@ needs-llvm-components: amdgpu
+#![feature(no_core, abi_gpu_kernel)]
+#![no_core]
+#![allow(improper_gpu_kernel_arg)]
+
+extern crate minicore;
+use minicore::num::Complex;
+
+// Tests from llvm-project/clang/test/CodeGenOpenCL/amdgpu-abi-struct-coerce.cl
+
+#[repr(C)]
+pub struct SingleElementStructArg {
+    i: i32,
+}
+
+#[repr(C)]
+pub struct NestedSingleElementStructArg {
+    i: SingleElementStructArg,
+}
+
+#[repr(C)]
+pub struct StructArg {
+    i1: i32,
+    f: f32,
+    i2: i32,
+}
+
+#[repr(C)]
+pub struct StructPaddingArg {
+    i1: i8,
+    f: i64,
+}
+
+#[repr(C)]
+pub struct StructOfArraysArg {
+    i1: [i32; 2],
+    f1: f32,
+    i2: [i32; 4],
+    f2: [f32; 3],
+    i3: i32,
+}
+
+#[repr(C)]
+pub struct StructOfStructsArg {
+    i1: i32,
+    f1: f32,
+    s1: StructArg,
+    i2: i32,
+}
+
+#[repr(C)]
+pub union U {
+    b1: i32,
+    b2: f32,
+}
+
+#[repr(C)]
+pub struct SingleArrayElementStructArg {
+    i: [i32; 4],
+}
+
+#[repr(C)]
+pub struct SingleStructElementStructArgInner {
+    i: i32,
+    b: i64,
+}
+
+#[repr(C)]
+pub struct SingleStructElementStructArg {
+    s: SingleStructElementStructArgInner,
+}
+
+#[repr(C)]
+pub struct DifferentSizeTypePair {
+    l: i64,
+    i: i32,
+}
+
+// CHECK: define amdgpu_kernel void @kernel_single_element_struct_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(4) {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_single_element_struct_arg(_: SingleElementStructArg) {}
+
+// CHECK: define amdgpu_kernel void @kernel_nested_single_element_struct_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(4) {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_nested_single_element_struct_arg(
+    _: NestedSingleElementStructArg,
+) {
+}
+
+// CHECK: define amdgpu_kernel void @kernel_struct_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(12) {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_struct_arg(_: StructArg) {}
+
+// CHECK: define amdgpu_kernel void @kernel_struct_padding_arg(i8 noundef {{%.+}}, i64 noundef {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_struct_padding_arg(_: StructPaddingArg) {}
+
+// CHECK: define amdgpu_kernel void @kernel_struct_of_arrays_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(44) {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_struct_of_arrays_arg(_: StructOfArraysArg) {}
+
+// CHECK: define amdgpu_kernel void @kernel_struct_of_structs_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(24) {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_struct_of_structs_arg(_: StructOfStructsArg) {}
+
+// CHECK: define amdgpu_kernel void @test_kernel_union_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(4) {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn test_kernel_union_arg(_: U) {}
+
+// CHECK: define amdgpu_kernel void @kernel_single_array_element_struct_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(16) {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_single_array_element_struct_arg(_: SingleArrayElementStructArg) {}
+
+// CHECK: define amdgpu_kernel void @kernel_single_struct_element_struct_arg(i32 noundef {{%.+}}, i64 noundef {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_single_struct_element_struct_arg(
+    _: SingleStructElementStructArg,
+) {
+}
+
+// CHECK: define amdgpu_kernel void @kernel_different_size_type_pair_arg(i64 noundef {{%.+}}, i32 noundef {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_different_size_type_pair_arg(_: DifferentSizeTypePair) {}
+
+// CHECK: define amdgpu_kernel void @kernel_complex(float noundef {{%.+}}, float noundef {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_complex(_: Complex<f32>) {}
+
+// CHECK: define amdgpu_kernel void @kernel_slice(ptr noalias nofree noundef nonnull readonly align 4 captures(none) {{%.+}}, i64 noundef range(i64 0, 2305843009213693952) {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_slice(_: &[u32]) {}

From c4982542b6f702e25d3cebb55db4c2e29ea51662 Mon Sep 17 00:00:00 2001
From: Flakebi <flakebi@t-online.de>
Date: Tue, 15 Sep 2026 10:33:14 +0200
Subject: [PATCH 05/28] Properly implement the gpu-kernel ABI for amdgpu
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add support to pass structs, arrays and vectors to amdgpu kernels.
Scalars and vectors are taken by value, aggregates are passed by byref
pointers. Structs containing a single scalar/vector are handled like
a scalar.

Judging from clang tests, nvptx seems to do somewhat the same, just
using byval instead of byref: https://github.com/llvm/llvm-project/blob/3a8affeef4da19d39191aac316e189eca3214a8c/clang/test/CodeGenCUDA/kernel-args.cu

I tested a couple of the lit test signatures on real hardware and it
seems to work fine. Given the relatively simple implementation, I hope
this amount of testing is enough (the C calling convention seems like
a worse fit for Rust’s current ABI code, it’s still giving me headaches).
---
 compiler/rustc_abi/src/lib.rs                |  4 +
 compiler/rustc_target/src/callconv/amdgpu.rs | 82 ++++++++++++++----
 tests/codegen-llvm/amdgpu-abi/struct-abi.rs  | 88 ++++++++++++++++----
 3 files changed, 139 insertions(+), 35 deletions(-)

diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs
index b056fdc73d40b..79c0309c60aa3 100644
--- a/compiler/rustc_abi/src/lib.rs
+++ b/compiler/rustc_abi/src/lib.rs
@@ -1772,6 +1772,10 @@ pub struct AddressSpace(pub u32);
 impl AddressSpace {
     /// LLVM's `0` address space.
     pub const ZERO: Self = AddressSpace(0);
+    /// The address space for constant memory on nvptx and amdgpu.
+    /// This address space is used e.g. for kernel arguments that are constant throughout the
+    /// execution.
+    pub const GPU_CONSTANT: Self = AddressSpace(4);
     /// The address space for workgroup memory on nvptx and amdgpu.
     /// See e.g. the `gpu_launch_sized_workgroup_mem` intrinsic for details.
     pub const GPU_WORKGROUP: Self = AddressSpace(3);
diff --git a/compiler/rustc_target/src/callconv/amdgpu.rs b/compiler/rustc_target/src/callconv/amdgpu.rs
index 98ab3ce8eb746..7a9eeaba19c96 100644
--- a/compiler/rustc_target/src/callconv/amdgpu.rs
+++ b/compiler/rustc_target/src/callconv/amdgpu.rs
@@ -1,25 +1,60 @@
-use rustc_abi::{HasDataLayout, TyAbiInterface};
+use rustc_abi::{
+    AddressSpace, BackendRepr, CanonAbi, HasDataLayout, Reg, RegKind, TyAbiInterface, TyAndLayout,
+};
 
-use crate::callconv::{ArgAbi, FnAbi};
+use crate::callconv::{FnAbi, Uniform};
 
-fn classify_ret<'a, Ty, C>(_cx: &C, ret: &mut ArgAbi<'a, Ty>)
-where
-    Ty: TyAbiInterface<'a, C> + Copy,
-    C: HasDataLayout,
-{
-    ret.extend_integer_width_to(32);
-}
+// For reference, see llvm-project/clang/lib/CodeGen/Targets/AMDGPU.cpp
 
-fn classify_arg<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>)
+/// If the given type is a (potentially nested) struct containing a single scalar, return
+/// a `Uniform` for the contained, single element.
+fn single_element_struct_to_reg<'a, Ty, C>(cx: &C, ty: TyAndLayout<'a, Ty>) -> Option<Uniform>
 where
     Ty: TyAbiInterface<'a, C> + Copy,
     C: HasDataLayout,
 {
-    if arg.layout.pass_indirectly_in_non_rustic_abis(cx) {
-        arg.make_indirect();
-        return;
+    assert!(ty.is_aggregate(), "Only handles aggregate types");
+    if ty.layout.fields.count() != 1 {
+        return None;
+    }
+    let field = ty.field(cx, 0);
+    match field.backend_repr {
+        BackendRepr::SimdScalableVector { .. } => panic!("scalable vectors are unsupported"),
+        BackendRepr::Scalar(_) => {
+            // Check that the size is the same as the size for ty, so no extra padding
+            let size = field.layout.size.bytes();
+            if ty.layout.size.bytes() != size {
+                return None;
+            }
+
+            // clang passes the inner type directly, we emulate it with fitting integer types
+            match size {
+                1 => Some(Uniform::new(Reg::i8(), field.layout.size)),
+                2 => Some(Uniform::new(Reg::i16(), field.layout.size)),
+                4 => Some(Uniform::new(Reg::i32(), field.layout.size)),
+                8 => Some(Uniform::new(Reg::i64(), field.layout.size)),
+                16 => Some(Uniform::new(Reg::i128(), field.layout.size)),
+                s => panic!("Unhandled scalar of size {s} in amdgpu gpu-kernel ABI"),
+            }
+        }
+        BackendRepr::SimdVector { element, .. } => {
+            // Check that the size is the same as the size for ty, so no extra padding
+            let size = field.layout.size.bytes();
+            if ty.layout.size.bytes() != size {
+                return None;
+            }
+
+            // clang passes the inner type directly, we emulate it with a vector of the same type.
+            // The size is rounded up to the size of the complete type (including alignment).
+            let reg = Reg {
+                kind: RegKind::Vector { hint_vector_elem: element.primitive() },
+                size: field.layout.size,
+            };
+            Some(Uniform::new(reg, field.layout.size))
+        }
+        BackendRepr::Memory { .. } => single_element_struct_to_reg(cx, field),
+        BackendRepr::ScalarPair { .. } => None,
     }
-    arg.extend_integer_width_to(32);
 }
 
 pub(crate) fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>)
@@ -27,14 +62,25 @@ where
     Ty: TyAbiInterface<'a, C> + Copy,
     C: HasDataLayout,
 {
-    if !fn_abi.ret.is_ignore() {
-        classify_ret(cx, &mut fn_abi.ret);
-    }
+    // Kernels cannot return values, so do not handle return types
 
+    // Try to fill first registers with values and pass by_ref pointers for later indirect arguments
     for arg in fn_abi.args.iter_mut() {
         if arg.is_ignore() {
             continue;
         }
-        classify_arg(cx, arg);
+        if fn_abi.conv == CanonAbi::GpuKernel {
+            if arg.layout.is_aggregate() {
+                if let Some(uniform) = single_element_struct_to_reg(cx, arg.layout) {
+                    // Single element structs are passed directly as the inner type
+                    arg.cast_to(uniform);
+                } else {
+                    // All other aggregates are passed as by_ref pointer in the constant address space
+                    arg.pass_amdgpu_kernel_arg(Some(AddressSpace::GPU_CONSTANT));
+                }
+            }
+        } else {
+            // FIXME: C ABI is not yet implemented
+        }
     }
 }
diff --git a/tests/codegen-llvm/amdgpu-abi/struct-abi.rs b/tests/codegen-llvm/amdgpu-abi/struct-abi.rs
index bf51cbfa7f7a0..bc83f6510a6fe 100644
--- a/tests/codegen-llvm/amdgpu-abi/struct-abi.rs
+++ b/tests/codegen-llvm/amdgpu-abi/struct-abi.rs
@@ -1,7 +1,7 @@
 //@ add-minicore
 //@ compile-flags: --crate-type=rlib --target=amdgcn-amd-amdhsa -Ctarget-cpu=gfx900 -Copt-level=3
 //@ needs-llvm-components: amdgpu
-#![feature(no_core, abi_gpu_kernel)]
+#![feature(no_core, abi_gpu_kernel, repr_simd)]
 #![no_core]
 #![allow(improper_gpu_kernel_arg)]
 
@@ -10,14 +10,32 @@ use minicore::num::Complex;
 
 // Tests from llvm-project/clang/test/CodeGenOpenCL/amdgpu-abi-struct-coerce.cl
 
+#[repr(simd)]
+pub struct I8X2([i8; 2]);
+
+#[repr(simd)]
+pub struct I16X2([i16; 2]);
+
+#[repr(simd)]
+pub struct I16X3([i16; 3]);
+
+#[repr(simd)]
+pub struct I16X4([i16; 4]);
+
+#[repr(simd)]
+pub struct I32X3([i32; 3]);
+
+#[repr(simd)]
+pub struct I32X4([i32; 4]);
+
 #[repr(C)]
-pub struct SingleElementStructArg {
-    i: i32,
+pub struct SingleElementStructArg<T> {
+    i: T,
 }
 
 #[repr(C)]
 pub struct NestedSingleElementStructArg {
-    i: SingleElementStructArg,
+    i: SingleElementStructArg<i32>,
 }
 
 #[repr(C)]
@@ -78,56 +96,92 @@ pub struct DifferentSizeTypePair {
     i: i32,
 }
 
-// CHECK: define amdgpu_kernel void @kernel_single_element_struct_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(4) {{%.+}})
+// CHECK: define amdgpu_kernel void @kernel_single_element_struct_arg(i32 %0)
 #[no_mangle]
-pub extern "gpu-kernel" fn kernel_single_element_struct_arg(_: SingleElementStructArg) {}
+pub extern "gpu-kernel" fn kernel_single_element_struct_arg(_: SingleElementStructArg<i32>) {}
 
-// CHECK: define amdgpu_kernel void @kernel_nested_single_element_struct_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(4) {{%.+}})
+// CHECK: define amdgpu_kernel void @kernel_nested_single_element_struct_arg(i32 %0)
 #[no_mangle]
 pub extern "gpu-kernel" fn kernel_nested_single_element_struct_arg(
     _: NestedSingleElementStructArg,
 ) {
 }
 
-// CHECK: define amdgpu_kernel void @kernel_struct_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(12) {{%.+}})
+// CHECK: define amdgpu_kernel void @kernel_struct_arg(ptr addrspace(4) noalias nofree noundef readnone byref([12 x i8]) align 4 captures(none) dereferenceable(12) {{%.+}})
 #[no_mangle]
 pub extern "gpu-kernel" fn kernel_struct_arg(_: StructArg) {}
 
-// CHECK: define amdgpu_kernel void @kernel_struct_padding_arg(i8 noundef {{%.+}}, i64 noundef {{%.+}})
+// CHECK: define amdgpu_kernel void @kernel_struct_padding_arg(ptr addrspace(4) noalias nofree noundef readnone byref([16 x i8]) align 8 captures(none) dereferenceable(16) {{%.+}})
 #[no_mangle]
 pub extern "gpu-kernel" fn kernel_struct_padding_arg(_: StructPaddingArg) {}
 
-// CHECK: define amdgpu_kernel void @kernel_struct_of_arrays_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(44) {{%.+}})
+// CHECK: define amdgpu_kernel void @kernel_struct_of_arrays_arg(ptr addrspace(4) noalias nofree noundef readnone byref([44 x i8]) align 4 captures(none) dereferenceable(44) {{%.+}})
 #[no_mangle]
 pub extern "gpu-kernel" fn kernel_struct_of_arrays_arg(_: StructOfArraysArg) {}
 
-// CHECK: define amdgpu_kernel void @kernel_struct_of_structs_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(24) {{%.+}})
+// CHECK: define amdgpu_kernel void @kernel_struct_of_structs_arg(ptr addrspace(4) noalias nofree noundef readnone byref([24 x i8]) align 4 captures(none) dereferenceable(24) {{%.+}})
 #[no_mangle]
 pub extern "gpu-kernel" fn kernel_struct_of_structs_arg(_: StructOfStructsArg) {}
 
-// CHECK: define amdgpu_kernel void @test_kernel_union_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(4) {{%.+}})
+// CHECK: define amdgpu_kernel void @test_kernel_union_arg(ptr addrspace(4) noalias nofree noundef readnone byref([4 x i8]) align 4 captures(none) dereferenceable(4) {{%.+}})
 #[no_mangle]
 pub extern "gpu-kernel" fn test_kernel_union_arg(_: U) {}
 
-// CHECK: define amdgpu_kernel void @kernel_single_array_element_struct_arg(ptr noalias nofree noundef readnone align 4 captures(none) dead_on_return dereferenceable(16) {{%.+}})
+// CHECK: define amdgpu_kernel void @kernel_single_array_element_struct_arg(ptr addrspace(4) noalias nofree noundef readnone byref([16 x i8]) align 4 captures(none) dereferenceable(16) {{%.+}})
 #[no_mangle]
 pub extern "gpu-kernel" fn kernel_single_array_element_struct_arg(_: SingleArrayElementStructArg) {}
 
-// CHECK: define amdgpu_kernel void @kernel_single_struct_element_struct_arg(i32 noundef {{%.+}}, i64 noundef {{%.+}})
+// CHECK: define amdgpu_kernel void @kernel_single_struct_element_struct_arg(ptr addrspace(4) noalias nofree noundef readnone byref([16 x i8]) align 8 captures(none) dereferenceable(16) {{%.+}})
 #[no_mangle]
 pub extern "gpu-kernel" fn kernel_single_struct_element_struct_arg(
     _: SingleStructElementStructArg,
 ) {
 }
 
-// CHECK: define amdgpu_kernel void @kernel_different_size_type_pair_arg(i64 noundef {{%.+}}, i32 noundef {{%.+}})
+// CHECK: define amdgpu_kernel void @kernel_different_size_type_pair_arg(ptr addrspace(4) noalias nofree noundef readnone byref([16 x i8]) align 8 captures(none) dereferenceable(16) {{%.+}})
 #[no_mangle]
 pub extern "gpu-kernel" fn kernel_different_size_type_pair_arg(_: DifferentSizeTypePair) {}
 
-// CHECK: define amdgpu_kernel void @kernel_complex(float noundef {{%.+}}, float noundef {{%.+}})
+// CHECK: define amdgpu_kernel void @kernel_complex(ptr addrspace(4) noalias nofree noundef readnone byref([8 x i8]) align 4 captures(none) dereferenceable(8) {{%.+}})
 #[no_mangle]
 pub extern "gpu-kernel" fn kernel_complex(_: Complex<f32>) {}
 
-// CHECK: define amdgpu_kernel void @kernel_slice(ptr noalias nofree noundef nonnull readonly align 4 captures(none) {{%.+}}, i64 noundef range(i64 0, 2305843009213693952) {{%.+}})
+// CHECK: define amdgpu_kernel void @kernel_slice(ptr addrspace(4) noalias nofree noundef readnone byref([16 x i8]) align 8 captures(none) dereferenceable(16) {{%.+}})
 #[no_mangle]
 pub extern "gpu-kernel" fn kernel_slice(_: &[u32]) {}
+
+// CHECK: define amdgpu_kernel void @kernel_i64(i64 noundef {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_i64(_: i64) {}
+
+// CHECK: define amdgpu_kernel void @kernel_i64_struct(i64 {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_i64_struct(_: SingleElementStructArg<i64>) {}
+
+// CHECK: define amdgpu_kernel void @kernel_i128_struct(i128 {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_i128_struct(_: SingleElementStructArg<i128>) {}
+
+// CHECK: define amdgpu_kernel void @kernel_i8x2_struct(<2 x i8> {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_i8x2_struct(_: SingleElementStructArg<I8X2>) {}
+
+// CHECK: define amdgpu_kernel void @kernel_i16x2_struct(<2 x i16> {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_i16x2_struct(_: SingleElementStructArg<I16X2>) {}
+
+// CHECK: define amdgpu_kernel void @kernel_i16x3_struct(<4 x i16> {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_i16x3_struct(_: SingleElementStructArg<I16X3>) {}
+
+// CHECK: define amdgpu_kernel void @kernel_i16x4_struct(<4 x i16> {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_i16x4_struct(_: SingleElementStructArg<I16X4>) {}
+
+// CHECK: define amdgpu_kernel void @kernel_i32x3_struct(<4 x i32> {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_i32x3_struct(_: SingleElementStructArg<I32X3>) {}
+
+// CHECK: define amdgpu_kernel void @kernel_i32x4_struct(<4 x i32> {{%.+}})
+#[no_mangle]
+pub extern "gpu-kernel" fn kernel_i32x4_struct(_: SingleElementStructArg<I32X4>) {}

From 653076c76960316dc6a88a88c020bb07072a35fe Mon Sep 17 00:00:00 2001
From: khyperia <953151+khyperia@users.noreply.github.com>
Date: Fri, 18 Sep 2026 09:15:10 +0200
Subject: [PATCH 06/28] yeet AliasConstKind::opt_def_id

---
 .../rustc_hir_analysis/src/check/check.rs     | 20 ++++++--
 .../src/hir_ty_lowering/mod.rs                | 48 +++++++++----------
 compiler/rustc_middle/src/thir.rs             |  2 +-
 .../rustc_middle/src/ty/abstract_const.rs     | 11 +++--
 .../src/thir/pattern/check_match.rs           |  3 +-
 .../src/thir/pattern/const_to_pat.rs          |  2 +-
 .../src/unstable/convert/stable/ty.rs         | 10 ++--
 compiler/rustc_type_ir/src/const_kind.rs      | 10 ----
 .../mgca/inherent-alias-default.rs            | 15 ++++++
 tests/ui/thir-print/str-patterns.stdout       |  4 +-
 10 files changed, 77 insertions(+), 48 deletions(-)
 create mode 100644 tests/ui/const-generics/mgca/inherent-alias-default.rs

diff --git a/compiler/rustc_hir_analysis/src/check/check.rs b/compiler/rustc_hir_analysis/src/check/check.rs
index 705bb780a3ecf..52b69e6050a32 100644
--- a/compiler/rustc_hir_analysis/src/check/check.rs
+++ b/compiler/rustc_hir_analysis/src/check/check.rs
@@ -775,9 +775,23 @@ pub(crate) fn check_item_type(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Result<(),
                 if has_default {
                     // need to store default and type of default
                     let ct = tcx.const_param_default(param.def_id).skip_binder();
-                    if let ty::ConstKind::Alias(_, alias_const) = ct.kind()
-                        && let Some(def_id) = alias_const.kind.opt_def_id()
-                    {
+                    if let ty::ConstKind::Alias(_, alias_const) = ct.kind() {
+                        let def_id = match alias_const.kind {
+                            ty::AliasConstKind::Projection { def_id } => def_id,
+                            ty::AliasConstKind::InherentSelf { def_id } => {
+                                // NOTE: typically, InherentSelf is illegal to pass to type_of,
+                                // because the generic args are incorrect (type_of expects impl-form
+                                // arguments). However, we are just checking ensure_ok().type_of(),
+                                // we are not instantiating the result, so it's OK here.
+                                def_id
+                            }
+                            ty::AliasConstKind::InherentImpl { .. } => span_bug!(
+                                tcx.def_span(param.def_id),
+                                "const_param_default should return an unnormalized constant, which should always be InherentSelf, not InherentImpl"
+                            ),
+                            ty::AliasConstKind::Free { def_id } => def_id,
+                            ty::AliasConstKind::Anon { def_id } => def_id,
+                        };
                         tcx.ensure_ok().type_of(def_id);
                     }
                 }
diff --git a/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs b/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs
index e89caa6aeff8c..ecfe5d3c2c7c2 100644
--- a/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs
+++ b/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs
@@ -1480,9 +1480,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         )? {
             TypeRelativePath::AssocItem(alias_term) => {
                 let alias_ct = alias_term.expect_ct();
-                if let Some(def_id) = alias_ct.kind.opt_def_id() {
-                    self.check_const_item_in_type_system(def_id, span)?;
-                }
+                self.check_const_item_in_type_system(alias_ct.kind, span)?;
                 let ct = Const::new_alias(tcx, ty::IsRigid::No, alias_ct);
                 let ct = self.check_param_uses_if_mcg(ct, span, false);
                 Ok(ct)
@@ -1948,13 +1946,10 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
             item_segment,
             ty::AssocTag::Const,
         )?;
-        self.check_const_item_in_type_system(item_def_id, span)?;
-        let alias_const = ty::AliasConst::new(
-            tcx,
-            ty::AliasConstKind::Projection { def_id: item_def_id },
-            item_args,
-        );
-        Ok(Const::new_alias(tcx, ty::IsRigid::No, alias_const))
+        let kind = ty::AliasConstKind::Projection { def_id: item_def_id };
+        self.check_const_item_in_type_system(kind, span)?;
+        let alias = ty::AliasConst::new(tcx, kind, item_args);
+        Ok(Const::new_alias(tcx, ty::IsRigid::No, alias))
     }
 
     /// Lower a [resolved][hir::QPath::Resolved] (type-level) associated item path.
@@ -2879,7 +2874,8 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
                 self.lower_const_param(def_id, hir_id)
             }
             Res::Def(DefKind::Const, did) => {
-                if let Err(guar) = self.check_const_item_in_type_system(did, span) {
+                let kind = ty::AliasConstKind::Free { def_id: did };
+                if let Err(guar) = self.check_const_item_in_type_system(kind, span) {
                     return Const::new_error(self.tcx(), guar);
                 }
 
@@ -2888,11 +2884,8 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
                 let _ = self
                     .prohibit_generic_args(leading_segments.iter(), GenericsArgsErrExtend::None);
                 let args = self.lower_generic_args_of_path_segment(span, did, segment);
-                ty::Const::new_alias(
-                    tcx,
-                    ty::IsRigid::No,
-                    ty::AliasConst::new(tcx, ty::AliasConstKind::Free { def_id: did }, args),
-                )
+                let alias = ty::AliasConst::new(tcx, kind, args);
+                ty::Const::new_alias(tcx, ty::IsRigid::No, alias)
             }
             Res::Def(kind @ DefKind::Ctor(ctor_of, CtorKind::Const), did) => {
                 assert_eq!(opt_self_ty, None);
@@ -3126,18 +3119,27 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
     /// `def_id` is a const item used in the type system. Checks if that's OK.
     fn check_const_item_in_type_system(
         &self,
-        def_id: DefId,
+        alias_const: ty::AliasConstKind<'tcx>,
         span: Span,
     ) -> Result<(), ErrorGuaranteed> {
         let tcx = self.tcx();
-        if tcx.features().generic_const_args() || tcx.is_direct_const(def_id) {
+        if tcx.features().generic_const_args() || alias_const.is_direct_const(tcx) {
             Ok(())
         } else {
             let mut err = self
                 .dcx()
                 .struct_span_err(span, "use of `const` in the type system not marked as direct");
-            if let Some(local_def_id) = def_id.as_local() {
-                if let Some(body_id) = tcx.hir_node_by_def_id(local_def_id).body_id() {
+            let hir_node = match alias_const {
+                ty::AliasConstKind::Projection { def_id }
+                | ty::AliasConstKind::InherentSelf { def_id }
+                | ty::AliasConstKind::InherentImpl { def_id }
+                | ty::AliasConstKind::Free { def_id }
+                | ty::AliasConstKind::Anon { def_id } => {
+                    def_id.as_local().map(|id| tcx.hir_node_by_def_id(id))
+                }
+            };
+            if let Some(hir_node) = hir_node {
+                if let Some(body_id) = hir_node.body_id() {
                     let body_span = tcx.hir_body(body_id).value.span;
 
                     err.multipart_suggestion(
@@ -3148,10 +3150,8 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
                         ],
                         Applicability::MaybeIncorrect,
                     );
-                } else if let DefKind::AssocConst = tcx.def_kind(def_id)
-                    && let DefKind::Trait = tcx.def_kind(tcx.parent(def_id))
-                {
-                    let node = tcx.hir_node_by_def_id(local_def_id).expect_trait_item();
+                } else if let ty::AliasConstKind::Projection { .. } = alias_const {
+                    let node = hir_node.expect_trait_item();
                     let sp = node.span.shrink_to_lo();
                     err.span_suggestion_verbose(
                         sp,
diff --git a/compiler/rustc_middle/src/thir.rs b/compiler/rustc_middle/src/thir.rs
index fe7b1bf493051..b20dfe68d98e2 100644
--- a/compiler/rustc_middle/src/thir.rs
+++ b/compiler/rustc_middle/src/thir.rs
@@ -661,7 +661,7 @@ pub struct PatExtra<'tcx> {
     ///
     /// This is used by some diagnostics for non-exhaustive matches, to map
     /// the pattern node back to the `DefId` of its original constant.
-    pub expanded_const: Option<DefId>,
+    pub expanded_const: Option<ty::AliasConstKind<'tcx>>,
 
     /// User-written types that must be preserved into MIR so that they can be
     /// checked.
diff --git a/compiler/rustc_middle/src/ty/abstract_const.rs b/compiler/rustc_middle/src/ty/abstract_const.rs
index 2853c43ae079d..2227841923514 100644
--- a/compiler/rustc_middle/src/ty/abstract_const.rs
+++ b/compiler/rustc_middle/src/ty/abstract_const.rs
@@ -52,9 +52,14 @@ impl<'tcx> TyCtxt<'tcx> {
             }
             fn fold_const(&mut self, c: Const<'tcx>) -> Const<'tcx> {
                 let ct = match c.kind() {
-                    ty::ConstKind::Alias(_, alias_const)
-                        if let Some(def_id) = alias_const.kind.opt_def_id() =>
-                    {
+                    ty::ConstKind::Alias(_, alias_const) => {
+                        let def_id = match alias_const.kind {
+                            ty::AliasConstKind::Projection { def_id }
+                            | ty::AliasConstKind::InherentSelf { def_id }
+                            | ty::AliasConstKind::InherentImpl { def_id }
+                            | ty::AliasConstKind::Free { def_id }
+                            | ty::AliasConstKind::Anon { def_id } => def_id,
+                        };
                         match self.tcx.thir_abstract_const(def_id) {
                             Err(e) => ty::Const::new_error(self.tcx, e),
                             Ok(Some(bac)) => {
diff --git a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
index befe4d67253a3..3414ee751585c 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
@@ -1229,8 +1229,7 @@ fn is_const_pat_that_looks_like_binding<'tcx>(tcx: TyCtxt<'tcx>, pat: &Pat<'tcx>
     // The pattern must be a named constant, and the name that appears in
     // the pattern's source text must resemble a plain identifier without any
     // `::` namespace separators or other non-identifier characters.
-    if let Some(def_id) = try { pat.extra.as_deref()?.expanded_const? }
-        && tcx.def_kind(def_id) == DefKind::Const
+    if let ty::AliasConstKind::Free { def_id } = pat.extra.as_deref()?.expanded_const?
         && let Ok(snippet) = tcx.sess.source_map().span_to_snippet(pat.span)
         && snippet.chars().all(|c| c.is_alphanumeric() || c == '_')
     {
diff --git a/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
index 55eef6006f278..e24ef5ea5b52d 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
@@ -224,7 +224,7 @@ impl<'tcx> ConstToPat<'tcx> {
 
         // Mark the pattern to indicate that it is the result of lowering a named
         // constant. This is used for diagnostics.
-        thir_pat.extra.get_or_insert_default().expanded_const = alias_const.kind.opt_def_id();
+        thir_pat.extra.get_or_insert_default().expanded_const = Some(alias_const.kind);
         thir_pat
     }
 
diff --git a/compiler/rustc_public/src/unstable/convert/stable/ty.rs b/compiler/rustc_public/src/unstable/convert/stable/ty.rs
index ad1aa1b47132a..c3eebccb6b762 100644
--- a/compiler/rustc_public/src/unstable/convert/stable/ty.rs
+++ b/compiler/rustc_public/src/unstable/convert/stable/ty.rs
@@ -557,9 +557,13 @@ impl<'tcx> Stable<'tcx> for ty::Const<'tcx> {
             }
             ty::ConstKind::Param(param) => crate::ty::TyConstKind::Param(param.stable(tables, cx)),
             ty::ConstKind::Alias(_, alias_const) => {
-                let Some(def_id) = alias_const.kind.opt_def_id() else {
-                    // FIXME: implement (both AliasTy and AliasConst will be needing this soon)
-                    panic!("non-defid alias consts are not supported by rustc_public at the moment")
+                // rustc_public must change its API once we introduce a variant without a def_id.
+                let def_id = match alias_const.kind {
+                    ty::AliasConstKind::Projection { def_id }
+                    | ty::AliasConstKind::InherentSelf { def_id }
+                    | ty::AliasConstKind::InherentImpl { def_id }
+                    | ty::AliasConstKind::Free { def_id }
+                    | ty::AliasConstKind::Anon { def_id } => def_id,
                 };
                 crate::ty::TyConstKind::Unevaluated(
                     tables.const_def(def_id),
diff --git a/compiler/rustc_type_ir/src/const_kind.rs b/compiler/rustc_type_ir/src/const_kind.rs
index dd0578610a2a0..f66322e034814 100644
--- a/compiler/rustc_type_ir/src/const_kind.rs
+++ b/compiler/rustc_type_ir/src/const_kind.rs
@@ -160,16 +160,6 @@ impl<I: Interner> AliasConstKind<I> {
             AliasConstKind::Anon { def_id } => interner.def_span(def_id.into()),
         }
     }
-
-    pub fn opt_def_id(self) -> Option<I::DefId> {
-        match self {
-            AliasConstKind::Projection { def_id } => Some(def_id.into()),
-            AliasConstKind::InherentSelf { def_id } => Some(def_id.into()),
-            AliasConstKind::InherentImpl { def_id } => Some(def_id.into()),
-            AliasConstKind::Free { def_id } => Some(def_id.into()),
-            AliasConstKind::Anon { def_id } => Some(def_id.into()),
-        }
-    }
 }
 
 rustc_index::newtype_index! {
diff --git a/tests/ui/const-generics/mgca/inherent-alias-default.rs b/tests/ui/const-generics/mgca/inherent-alias-default.rs
new file mode 100644
index 0000000000000..9ba6d1d15e856
--- /dev/null
+++ b/tests/ui/const-generics/mgca/inherent-alias-default.rs
@@ -0,0 +1,15 @@
+//@ check-pass
+//! rustc_hir_analysis::check_item_type does type_of() on the default value. This is wonky, because
+//! the generic args are in Self format at that point, not in impl format, so the result can't be
+//! used with the Self-format args. However, it does not instantiate the result, it just does
+//! ensure_ok(). This test just makes sure that codepath is hit in tests.
+#![feature(min_generic_const_args, inherent_associated_types)]
+
+struct Struct<T1, T2, T3>(T1, T2, T3);
+impl<T1, T2, T3> Struct<T1, T2, T3> {
+    const INHERENT: usize = core::direct_const_arg!(2);
+}
+
+struct WithDefault<const N: usize = { core::direct_const_arg!(Struct::<u8, u16, u32>::INHERENT) }>;
+
+fn main() {}
diff --git a/tests/ui/thir-print/str-patterns.stdout b/tests/ui/thir-print/str-patterns.stdout
index da1f86b8fc591..61bcbaef5029a 100644
--- a/tests/ui/thir-print/str-patterns.stdout
+++ b/tests/ui/thir-print/str-patterns.stdout
@@ -46,7 +46,9 @@ Thir {
                 extra: Some(
                     PatExtra {
                         expanded_const: Some(
-                            DefId(0:4 ~ str_patterns[fc71]::CONSTANT),
+                            Free {
+                                def_id: DefId(0:4 ~ str_patterns[fc71]::CONSTANT),
+                            },
                         ),
                         ascriptions: [],
                     },

From 9e7d4b22edf5c5fa303673c52f2985f7ea823748 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Tue, 1 Sep 2026 14:54:42 +1000
Subject: [PATCH 07/28] Don't double-allocate `OwnerInfo`

`into_owner_info` arena-allocates the created `OwnerInfo`.
`ItemLowerer::with_lctx` calls `into_owner_info` and then
re-arena-allocates the returned `OwnerInfo` (the reference, not the
entire struct). This commit removes the latter.
---
 compiler/rustc_ast_lowering/src/item.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/compiler/rustc_ast_lowering/src/item.rs b/compiler/rustc_ast_lowering/src/item.rs
index edf184f568b22..24d5a80c59fab 100644
--- a/compiler/rustc_ast_lowering/src/item.rs
+++ b/compiler/rustc_ast_lowering/src/item.rs
@@ -62,8 +62,7 @@ impl<'hir> ItemLowerer<'_, 'hir> {
 
         let item = f(&mut lctx);
 
-        let info = lctx.curr_owner.into_owner_info(self.tcx, item);
-        hir::MaybeOwner::Owner(lctx.arena.alloc(info))
+        hir::MaybeOwner::Owner(lctx.curr_owner.into_owner_info(self.tcx, item))
     }
 
     #[instrument(level = "debug", skip(self, c))]

From 9a03326dc02ee524716005615061b5494093bc93 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Tue, 1 Sep 2026 14:59:45 +1000
Subject: [PATCH 08/28] Convert some `&mut self` to `&self` in the lowerer

---
 compiler/rustc_ast_lowering/src/item.rs | 12 ++++++------
 compiler/rustc_ast_lowering/src/lib.rs  | 12 ++++++------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/compiler/rustc_ast_lowering/src/item.rs b/compiler/rustc_ast_lowering/src/item.rs
index 24d5a80c59fab..7fe5a089fe155 100644
--- a/compiler/rustc_ast_lowering/src/item.rs
+++ b/compiler/rustc_ast_lowering/src/item.rs
@@ -54,7 +54,7 @@ fn add_ty_alias_where_clause(
 
 impl<'hir> ItemLowerer<'_, 'hir> {
     fn with_lctx(
-        &mut self,
+        &self,
         owner: NodeId,
         f: impl FnOnce(&mut LoweringContext<'_, 'hir>) -> hir::OwnerNode<'hir>,
     ) -> hir::MaybeOwner<'hir> {
@@ -66,7 +66,7 @@ impl<'hir> ItemLowerer<'_, 'hir> {
     }
 
     #[instrument(level = "debug", skip(self, c))]
-    pub(super) fn lower_crate(&mut self, c: &Crate) -> hir::MaybeOwner<'hir> {
+    pub(super) fn lower_crate(&self, c: &Crate) -> hir::MaybeOwner<'hir> {
         self.with_lctx(CRATE_NODE_ID, |lctx| {
             debug_assert_eq!(lctx.curr_owner.owner_id(), CRATE_OWNER_ID);
             let module = lctx.lower_mod(&c.items, &c.spans);
@@ -76,19 +76,19 @@ impl<'hir> ItemLowerer<'_, 'hir> {
     }
 
     #[instrument(level = "debug", skip(self))]
-    pub(super) fn lower_item(&mut self, item: &Item) -> hir::MaybeOwner<'hir> {
+    pub(super) fn lower_item(&self, item: &Item) -> hir::MaybeOwner<'hir> {
         self.with_lctx(item.id, |lctx| hir::OwnerNode::Item(lctx.lower_item(item)))
     }
 
-    pub(super) fn lower_trait_item(&mut self, item: &AssocItem) -> hir::MaybeOwner<'hir> {
+    pub(super) fn lower_trait_item(&self, item: &AssocItem) -> hir::MaybeOwner<'hir> {
         self.with_lctx(item.id, |lctx| hir::OwnerNode::TraitItem(lctx.lower_trait_item(item)))
     }
 
-    pub(super) fn lower_impl_item(&mut self, item: &AssocItem) -> hir::MaybeOwner<'hir> {
+    pub(super) fn lower_impl_item(&self, item: &AssocItem) -> hir::MaybeOwner<'hir> {
         self.with_lctx(item.id, |lctx| hir::OwnerNode::ImplItem(lctx.lower_impl_item(item)))
     }
 
-    pub(super) fn lower_foreign_item(&mut self, item: &ForeignItem) -> hir::MaybeOwner<'hir> {
+    pub(super) fn lower_foreign_item(&self, item: &ForeignItem) -> hir::MaybeOwner<'hir> {
         self.with_lctx(item.id, |lctx| hir::OwnerNode::ForeignItem(lctx.lower_foreign_item(item)))
     }
 }
diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs
index 19c37f4a76065..eb64d5027cdba 100644
--- a/compiler/rustc_ast_lowering/src/lib.rs
+++ b/compiler/rustc_ast_lowering/src/lib.rs
@@ -783,7 +783,7 @@ fn lower_to_hir(tcx: TyCtxt<'_>, def_id: LocalDefId) -> hir::MaybeOwner<'_> {
         return fallback_to_ancestor(tcx.local_parent(def_id));
     };
 
-    let mut item_lowerer = item::ItemLowerer { tcx, resolver: &*resolver };
+    let item_lowerer = item::ItemLowerer { tcx, resolver: &*resolver };
 
     let item = match &node {
         // The item existed in the AST.
@@ -982,7 +982,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
     }
 
     #[instrument(level = "trace", skip(self))]
-    fn lower_res(&mut self, res: Res<NodeId>) -> Res {
+    fn lower_res(&self, res: Res<NodeId>) -> Res {
         let res: Result<Res, ()> = res.apply_id(|id| {
             let owner = self.curr_owner.owner_id();
             let local_id =
@@ -999,11 +999,11 @@ impl<'hir> LoweringContext<'_, 'hir> {
         res.unwrap_or(Res::Err)
     }
 
-    fn expect_full_res(&mut self, id: NodeId) -> Res<NodeId> {
+    fn expect_full_res(&self, id: NodeId) -> Res<NodeId> {
         self.get_partial_res(id).map_or(Res::Err, |pr| pr.expect_full_res())
     }
 
-    fn lower_import_res(&mut self, id: NodeId, span: Span) -> PerNS<Option<Res>> {
+    fn lower_import_res(&self, id: NodeId, span: Span) -> PerNS<Option<Res>> {
         debug_assert_eq!(id, self.curr_owner.owner.id);
         let per_ns = self.curr_owner.owner.import_res.map(|res| res.map(|res| self.lower_res(res)));
         if per_ns.is_empty() {
@@ -3058,7 +3058,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         }))
     }
 
-    fn lower_unsafe_source(&mut self, u: UnsafeSource) -> hir::UnsafeSource {
+    fn lower_unsafe_source(&self, u: UnsafeSource) -> hir::UnsafeSource {
         match u {
             CompilerGenerated => hir::UnsafeSource::CompilerGenerated,
             UserProvided => hir::UnsafeSource::UserProvided,
@@ -3066,7 +3066,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
     }
 
     fn lower_trait_bound_modifiers(
-        &mut self,
+        &self,
         modifiers: TraitBoundModifiers,
     ) -> hir::TraitBoundModifiers {
         let constness = match modifiers.constness {

From 990c292e4178cff8526a50aa47221936a5440feb Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Tue, 1 Sep 2026 15:07:48 +1000
Subject: [PATCH 09/28] Reduce the scope of a local

---
 compiler/rustc_ast_lowering/src/lib.rs | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs
index eb64d5027cdba..087182e43f95c 100644
--- a/compiler/rustc_ast_lowering/src/lib.rs
+++ b/compiler/rustc_ast_lowering/src/lib.rs
@@ -1647,8 +1647,8 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 self.lower_array_length_to_const_arg(length),
             ),
             TyKind::TraitObject(bounds, kind) => {
-                let mut lifetime_bound = None;
                 let (bounds, lifetime_bound) = self.with_dyn_type_scope(true, |this| {
+                    let mut lifetime_bound = None;
                     let bounds =
                         this.arena.alloc_from_iter(bounds.iter().filter_map(|bound| match bound {
                             // We can safely ignore constness here since AST validation
@@ -1681,9 +1681,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                                 None
                             }
                         }));
-                    let lifetime_bound =
-                        lifetime_bound.unwrap_or_else(|| this.elided_dyn_bound(t.span));
-                    (bounds, lifetime_bound)
+                    (bounds, lifetime_bound.unwrap_or_else(|| this.elided_dyn_bound(t.span)))
                 });
                 hir::TyKind::TraitObject(bounds, TaggedRef::new(lifetime_bound, *kind))
             }

From 9b817c55d6e7cd0bc232952bbc4661b833a9b601 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Tue, 1 Sep 2026 15:08:45 +1000
Subject: [PATCH 10/28] Rename `LoweringContext::current_item`

As `LoweringContext::current_item_span`, because it *is* a span.
---
 compiler/rustc_ast_lowering/src/expr.rs |  4 ++--
 compiler/rustc_ast_lowering/src/lib.rs  | 10 +++++-----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs
index 1c1b9a247f7a2..c3bc86a352644 100644
--- a/compiler/rustc_ast_lowering/src/expr.rs
+++ b/compiler/rustc_ast_lowering/src/expr.rs
@@ -1018,7 +1018,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     expr.span,
                     hir::ExprKind::Err(self.dcx().emit_err(AwaitOnlyInAsyncFnAndBlocks {
                         await_kw_span,
-                        item_span: self.current_item,
+                        item_span: self.current_item_span,
                     })),
                 );
                 return hir::ExprKind::Block(
@@ -1712,7 +1712,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
             }
             Some(hir::CoroutineKind::Coroutine(_)) => false,
             None => {
-                let suggestion = self.current_item.map(|s| s.shrink_to_lo());
+                let suggestion = self.current_item_span.map(|s| s.shrink_to_lo());
                 self.dcx().emit_err(YieldInClosure { span, suggestion });
                 self.coroutine_kind = Some(hir::CoroutineKind::Coroutine(Movability::Movable));
 
diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs
index 087182e43f95c..b9d6f478c9ac0 100644
--- a/compiler/rustc_ast_lowering/src/lib.rs
+++ b/compiler/rustc_ast_lowering/src/lib.rs
@@ -296,7 +296,7 @@ struct LoweringContext<'a, 'hir> {
 
     /// Used to get the current `fn`'s def span to point to when using `await`
     /// outside of an `async fn`.
-    current_item: Option<Span>,
+    current_item_span: Option<Span>,
 
     try_block_scope: TryBlockScope,
     loop_scope: Option<HirId>,
@@ -366,7 +366,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             is_in_dyn_type: false,
             coroutine_kind: None,
             task_context: None,
-            current_item: None,
+            current_item_span: None,
 
             move_expr_bindings: Vec::new(),
             lowering_move_expr_initializer: false,
@@ -1154,8 +1154,8 @@ impl<'hir> LoweringContext<'_, 'hir> {
     }
 
     fn with_new_scopes<T>(&mut self, scope_span: Span, f: impl FnOnce(&mut Self) -> T) -> T {
-        let current_item = self.current_item;
-        self.current_item = Some(scope_span);
+        let current_item_span = self.current_item_span;
+        self.current_item_span = Some(scope_span);
 
         let was_in_loop_condition = self.is_in_loop_condition;
         self.is_in_loop_condition = false;
@@ -1172,7 +1172,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
 
         self.is_in_loop_condition = was_in_loop_condition;
 
-        self.current_item = current_item;
+        self.current_item_span = current_item_span;
 
         ret
     }

From 3ee39bdf7cc213a7e4e7ff70d91948302f485dfc Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Tue, 1 Sep 2026 15:09:32 +1000
Subject: [PATCH 11/28] Fix an inconsistent comment

---
 compiler/rustc_ast_lowering/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs
index b9d6f478c9ac0..609644804b203 100644
--- a/compiler/rustc_ast_lowering/src/lib.rs
+++ b/compiler/rustc_ast_lowering/src/lib.rs
@@ -824,7 +824,7 @@ enum GenericArgsMode {
     ParenSugar,
     /// Allow RTN, don't allow paren sugar.
     ReturnTypeNotation,
-    // Error if parenthesized generics or RTN are encountered.
+    /// Error if parenthesized generics or RTN are encountered.
     Err,
     /// Silence errors when lowering generics. Only used with `Res::Err`.
     Silence,

From 6e7d014afa5e089a0ffab69c738ca635bfbfd963 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Tue, 1 Sep 2026 21:50:32 +1000
Subject: [PATCH 12/28] Inline and remove `lower_delim_args`

It's trivial and has a single call site.
---
 compiler/rustc_ast_lowering/src/item.rs | 2 +-
 compiler/rustc_ast_lowering/src/lib.rs  | 4 ----
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/compiler/rustc_ast_lowering/src/item.rs b/compiler/rustc_ast_lowering/src/item.rs
index 7fe5a089fe155..72f72d22fb13a 100644
--- a/compiler/rustc_ast_lowering/src/item.rs
+++ b/compiler/rustc_ast_lowering/src/item.rs
@@ -543,7 +543,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
             }
             ItemKind::MacroDef(ident, MacroDef { body, macro_rules, eii_declaration: _ }) => {
                 let ident = self.lower_ident(*ident);
-                let body = Box::new(self.lower_delim_args(body));
+                let body = body.clone();
                 let def_id = self.curr_owner.owner.def_id;
                 let def_kind = self.tcx.def_kind(def_id);
                 let DefKind::Macro(macro_kinds) = def_kind else {
diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs
index 609644804b203..16d098d1c4d75 100644
--- a/compiler/rustc_ast_lowering/src/lib.rs
+++ b/compiler/rustc_ast_lowering/src/lib.rs
@@ -1261,10 +1261,6 @@ impl<'hir> LoweringContext<'_, 'hir> {
         }
     }
 
-    fn lower_delim_args(&self, args: &DelimArgs) -> DelimArgs {
-        args.clone()
-    }
-
     /// Lower an associated item constraint.
     #[instrument(level = "debug", skip_all)]
     fn lower_assoc_item_constraint(

From 11b373edc96b4bc0f4cf2e1bd692bffbd5ac4dc4 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Wed, 2 Sep 2026 07:13:08 +1000
Subject: [PATCH 13/28] Eliminate `ItemLowerer`

It's just a thin wrapper around `tcx` and `resolver`. The `lower_*`
methods all have a single call site and can be inlined, and `with_lctx`
can just be a local fn within `lower_to_hir`.

This requires increasing the visibility of some
`LoweringContext::lower_*` methods that are now called outside of
`item.rs`.
---
 compiler/rustc_ast_lowering/src/item.rs | 65 +++----------------------
 compiler/rustc_ast_lowering/src/lib.rs  | 39 ++++++++++++---
 2 files changed, 39 insertions(+), 65 deletions(-)

diff --git a/compiler/rustc_ast_lowering/src/item.rs b/compiler/rustc_ast_lowering/src/item.rs
index 72f72d22fb13a..1073706499919 100644
--- a/compiler/rustc_ast_lowering/src/item.rs
+++ b/compiler/rustc_ast_lowering/src/item.rs
@@ -5,11 +5,8 @@ use rustc_errors::{E0570, ErrorGuaranteed, struct_span_code_err};
 use rustc_hir::attrs::{AttributeKind, EiiImplResolution};
 use rustc_hir::def::{DefKind, PerNS, Res};
 use rustc_hir::{
-    self as hir, CRATE_OWNER_ID, HirId, ImplItemImplKind, LifetimeSource, PredicateOrigin, Target,
-    find_attr,
+    self as hir, HirId, ImplItemImplKind, LifetimeSource, PredicateOrigin, Target, find_attr,
 };
-use rustc_middle::middle::resolve::ResolverAstLowering;
-use rustc_middle::ty::TyCtxt;
 use rustc_middle::ty::data_structures::IndexMap;
 use rustc_span::def_id::{DefId, LocalDefId};
 use rustc_span::edit_distance::find_best_match_for_name;
@@ -28,14 +25,9 @@ use super::{
 };
 use crate::diagnostics::{ConstComptimeFn, ResolvingRestrictionKind, RestrictionAncestorOnly};
 
-pub(super) struct ItemLowerer<'a, 'hir> {
-    pub(super) tcx: TyCtxt<'hir>,
-    pub(super) resolver: &'a ResolverAstLowering<'hir>,
-}
-
-/// When we have a ty alias we *may* have two where clauses. To give the best diagnostics, we set the span
-/// to the where clause that is preferred, if it exists. Otherwise, it sets the span to the other where
-/// clause if it exists.
+/// When we have a ty alias we *may* have two where clauses. To give the best diagnostics, we set
+/// the span to the where clause that is preferred, if it exists. Otherwise, it sets the span to
+/// the other where clause if it exists.
 fn add_ty_alias_where_clause(
     generics: &mut ast::Generics,
     after_where_clause: &ast::WhereClause,
@@ -52,47 +44,6 @@ fn add_ty_alias_where_clause(
         if before.0 || !after.0 { before } else { after };
 }
 
-impl<'hir> ItemLowerer<'_, 'hir> {
-    fn with_lctx(
-        &self,
-        owner: NodeId,
-        f: impl FnOnce(&mut LoweringContext<'_, 'hir>) -> hir::OwnerNode<'hir>,
-    ) -> hir::MaybeOwner<'hir> {
-        let mut lctx = LoweringContext::new(self.tcx, self.resolver, owner);
-
-        let item = f(&mut lctx);
-
-        hir::MaybeOwner::Owner(lctx.curr_owner.into_owner_info(self.tcx, item))
-    }
-
-    #[instrument(level = "debug", skip(self, c))]
-    pub(super) fn lower_crate(&self, c: &Crate) -> hir::MaybeOwner<'hir> {
-        self.with_lctx(CRATE_NODE_ID, |lctx| {
-            debug_assert_eq!(lctx.curr_owner.owner_id(), CRATE_OWNER_ID);
-            let module = lctx.lower_mod(&c.items, &c.spans);
-            lctx.lower_attrs(hir::CRATE_HIR_ID, &c.attrs, c.spans.inner_span, Target::Crate);
-            hir::OwnerNode::Crate(module)
-        })
-    }
-
-    #[instrument(level = "debug", skip(self))]
-    pub(super) fn lower_item(&self, item: &Item) -> hir::MaybeOwner<'hir> {
-        self.with_lctx(item.id, |lctx| hir::OwnerNode::Item(lctx.lower_item(item)))
-    }
-
-    pub(super) fn lower_trait_item(&self, item: &AssocItem) -> hir::MaybeOwner<'hir> {
-        self.with_lctx(item.id, |lctx| hir::OwnerNode::TraitItem(lctx.lower_trait_item(item)))
-    }
-
-    pub(super) fn lower_impl_item(&self, item: &AssocItem) -> hir::MaybeOwner<'hir> {
-        self.with_lctx(item.id, |lctx| hir::OwnerNode::ImplItem(lctx.lower_impl_item(item)))
-    }
-
-    pub(super) fn lower_foreign_item(&self, item: &ForeignItem) -> hir::MaybeOwner<'hir> {
-        self.with_lctx(item.id, |lctx| hir::OwnerNode::ForeignItem(lctx.lower_foreign_item(item)))
-    }
-}
-
 impl<'hir> LoweringContext<'_, 'hir> {
     pub(super) fn lower_mod(
         &mut self,
@@ -202,7 +153,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         }
     }
 
-    fn lower_item(&mut self, i: &Item) -> &'hir hir::Item<'hir> {
+    pub(super) fn lower_item(&mut self, i: &Item) -> &'hir hir::Item<'hir> {
         let owner_id = self.curr_owner.owner_id();
         let hir_id: HirId = owner_id.into();
         let vis_span = self.lower_span(i.vis.span);
@@ -729,7 +680,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         }
     }
 
-    fn lower_foreign_item(&mut self, i: &ForeignItem) -> &'hir hir::ForeignItem<'hir> {
+    pub(super) fn lower_foreign_item(&mut self, i: &ForeignItem) -> &'hir hir::ForeignItem<'hir> {
         let owner_id = self.curr_owner.owner_id();
         let hir_id: HirId = owner_id.into();
         let attrs =
@@ -910,7 +861,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         }
     }
 
-    fn lower_trait_item(&mut self, i: &AssocItem) -> &'hir hir::TraitItem<'hir> {
+    pub(super) fn lower_trait_item(&mut self, i: &AssocItem) -> &'hir hir::TraitItem<'hir> {
         let trait_item_def_id = self.curr_owner.owner_id();
         let hir_id: HirId = trait_item_def_id.into();
         let attrs = self.lower_attrs(
@@ -1159,7 +1110,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         ident
     }
 
-    fn lower_impl_item(&mut self, i: &AssocItem) -> &'hir hir::ImplItem<'hir> {
+    pub(super) fn lower_impl_item(&mut self, i: &AssocItem) -> &'hir hir::ImplItem<'hir> {
         let owner_id = self.curr_owner.owner_id();
         let hir_id: HirId = owner_id.into();
         let parent_id = self.tcx.local_parent(owner_id.def_id);
diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs
index 16d098d1c4d75..410823919c523 100644
--- a/compiler/rustc_ast_lowering/src/lib.rs
+++ b/compiler/rustc_ast_lowering/src/lib.rs
@@ -60,8 +60,9 @@ use rustc_hir::def_id::{DefId, LOCAL_CRATE, LocalDefId, LocalDefIdMap};
 use rustc_hir::definitions::PerParentDisambiguatorState;
 use rustc_hir::lints::DelayedLint;
 use rustc_hir::{
-    self as hir, AngleBrackets, ConstArg, GenericArg, HirId, ItemLocalMap, LifetimeSource,
-    LifetimeSyntax, MissingLifetimeKind, ParamName, Target, TraitCandidate, find_attr,
+    self as hir, AngleBrackets, CRATE_OWNER_ID, ConstArg, GenericArg, HirId, ItemLocalMap,
+    LifetimeSource, LifetimeSyntax, MissingLifetimeKind, ParamName, Target, TraitCandidate,
+    find_attr,
 };
 use rustc_index::{Idx, IndexSlice, IndexVec};
 use rustc_macros::extension;
@@ -783,15 +784,37 @@ fn lower_to_hir(tcx: TyCtxt<'_>, def_id: LocalDefId) -> hir::MaybeOwner<'_> {
         return fallback_to_ancestor(tcx.local_parent(def_id));
     };
 
-    let item_lowerer = item::ItemLowerer { tcx, resolver: &*resolver };
+    fn with_lctx<'hir>(
+        tcx: TyCtxt<'hir>,
+        resolver: &ResolverAstLowering<'hir>,
+        owner: NodeId,
+        f: impl FnOnce(&mut LoweringContext<'_, 'hir>) -> hir::OwnerNode<'hir>,
+    ) -> hir::MaybeOwner<'hir> {
+        let mut lctx = LoweringContext::new(tcx, resolver, owner);
+        let item = f(&mut lctx);
+        hir::MaybeOwner::Owner(lctx.curr_owner.into_owner_info(tcx, item))
+    }
 
     let item = match &node {
         // The item existed in the AST.
-        AstOwner::Crate(c) => item_lowerer.lower_crate(&c),
-        AstOwner::Item(item) => item_lowerer.lower_item(&item),
-        AstOwner::TraitItem(item) => item_lowerer.lower_trait_item(&item),
-        AstOwner::ImplItem(item) => item_lowerer.lower_impl_item(&item),
-        AstOwner::ForeignItem(item) => item_lowerer.lower_foreign_item(&item),
+        AstOwner::Crate(c) => with_lctx(tcx, &*resolver, CRATE_NODE_ID, |lctx| {
+            debug_assert_eq!(lctx.curr_owner.owner_id(), CRATE_OWNER_ID);
+            let module = lctx.lower_mod(&c.items, &c.spans);
+            lctx.lower_attrs(hir::CRATE_HIR_ID, &c.attrs, c.spans.inner_span, Target::Crate);
+            hir::OwnerNode::Crate(module)
+        }),
+        AstOwner::Item(item) => {
+            with_lctx(tcx, &*resolver, item.id, |lctx| hir::OwnerNode::Item(lctx.lower_item(item)))
+        }
+        AstOwner::TraitItem(item) => with_lctx(tcx, &*resolver, item.id, |lctx| {
+            hir::OwnerNode::TraitItem(lctx.lower_trait_item(item))
+        }),
+        AstOwner::ImplItem(item) => with_lctx(tcx, &*resolver, item.id, |lctx| {
+            hir::OwnerNode::ImplItem(lctx.lower_impl_item(item))
+        }),
+        AstOwner::ForeignItem(item) => with_lctx(tcx, &*resolver, item.id, |lctx| {
+            hir::OwnerNode::ForeignItem(lctx.lower_foreign_item(item))
+        }),
         AstOwner::NestedUseTree(owner_id) => fallback_to_ancestor(*owner_id),
         // The item existed in the AST, but is not a HIR owner.
         // Fetch the correct information from its parent.

From 501bf1db693d40f04047af21d2cae92dd87790e2 Mon Sep 17 00:00:00 2001
From: James Barford-Evans <james.barford-evans@arm.com>
Date: Fri, 18 Sep 2026 10:34:03 +0100
Subject: [PATCH 14/28] `f16b` Implementation and documentation

---
 compiler/rustc_attr_ir/src/lang_items.rs      |   1 +
 compiler/rustc_feature/src/builtin_attrs.rs   |   5 +
 compiler/rustc_feature/src/unstable.rs        |   4 +
 .../rustc_hir_analysis/src/check/check.rs     |   2 +
 compiler/rustc_interface/src/util.rs          |   4 +
 compiler/rustc_middle/src/ty/layout.rs        |   5 +
 compiler/rustc_span/src/symbol.rs             |   3 +
 compiler/rustc_ty_utils/src/layout.rs         |  10 +-
 library/core/Cargo.toml                       |   1 +
 library/core/src/num/bfloat.rs                | 183 ++++++++++++++++++
 library/core/src/num/mod.rs                   |   3 +
 .../src/language-features/f16b.md             |  11 ++
 12 files changed, 231 insertions(+), 1 deletion(-)
 create mode 100644 library/core/src/num/bfloat.rs
 create mode 100644 src/doc/unstable-book/src/language-features/f16b.md

diff --git a/compiler/rustc_attr_ir/src/lang_items.rs b/compiler/rustc_attr_ir/src/lang_items.rs
index c1ad05dc8e4a8..60890f7799290 100644
--- a/compiler/rustc_attr_ir/src/lang_items.rs
+++ b/compiler/rustc_attr_ir/src/lang_items.rs
@@ -281,6 +281,7 @@ language_item_table! {
     PartialEq,               sym::eq,                  eq_trait,                   Target::Trait,          GenericRequirement::Exact(1);
     PartialOrd,              sym::partial_ord,         partial_ord_trait,          Target::Trait,          GenericRequirement::Exact(1);
     CVoid,                   sym::c_void,              c_void,                     Target::Enum,           GenericRequirement::None;
+    F16B,                    sym::f16b,                f16b,                       Target::Struct,         GenericRequirement::Exact(0);
 
     Type,                    sym::type_info,           type_struct,                Target::Struct,         GenericRequirement::None;
     TypeGeneric,             sym::type_info_generic,   type_generic,               Target::Enum,         GenericRequirement::None;
diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs
index bb35a3281ccdc..0f96eb7f158ed 100644
--- a/compiler/rustc_feature/src/builtin_attrs.rs
+++ b/compiler/rustc_feature/src/builtin_attrs.rs
@@ -40,6 +40,11 @@ const GATED_CFGS: &[GatedCfg] = &[
         sym::cfg_target_has_reliable_f16_f128,
         Features::cfg_target_has_reliable_f16_f128,
     ),
+    (
+        sym::target_has_reliable_f16b,
+        sym::cfg_target_has_reliable_f16b,
+        Features::cfg_target_has_reliable_f16b,
+    ),
     (
         sym::target_has_reliable_f128,
         sym::cfg_target_has_reliable_f16_f128,
diff --git a/compiler/rustc_feature/src/unstable.rs b/compiler/rustc_feature/src/unstable.rs
index a7138a88ee399..94e3b77c6b575 100644
--- a/compiler/rustc_feature/src/unstable.rs
+++ b/compiler/rustc_feature/src/unstable.rs
@@ -227,6 +227,8 @@ declare_features! (
     (unstable, anonymous_lifetime_in_impl_trait, "1.63.0", None),
     /// Allows checking whether or not the backend correctly supports unstable float types.
     (internal, cfg_target_has_reliable_f16_f128, "1.88.0", None),
+    /// Allows checking whether or not the backend correctly supports the unstable `f16b` type.
+    (internal, cfg_target_has_reliable_f16b, "CURRENT_RUSTC_VERSION", None),
     /// Allows checking whether or not the target might have thread support.
     (internal, cfg_target_has_threads, "1.99.0", None),
     /// Allows identifying the `compiler_builtins` crate.
@@ -520,6 +522,8 @@ declare_features! (
     (unstable, f128, "1.78.0", Some(116909)),
     /// Allow using 16-bit (half precision) floating point numbers.
     (unstable, f16, "1.78.0", Some(116909)),
+    /// Allow using bfloat16 floating point numbers.
+    (unstable, f16b, "CURRENT_RUSTC_VERSION", Some(160630)),
     /// Allows the use of `#[ffi_const]` on foreign functions.
     (unstable, ffi_const, "1.45.0", Some(58328)),
     /// Allows the use of `#[ffi_pure]` on foreign functions.
diff --git a/compiler/rustc_hir_analysis/src/check/check.rs b/compiler/rustc_hir_analysis/src/check/check.rs
index 705bb780a3ecf..7cebf6dd16711 100644
--- a/compiler/rustc_hir_analysis/src/check/check.rs
+++ b/compiler/rustc_hir_analysis/src/check/check.rs
@@ -1592,6 +1592,8 @@ fn check_scalable_vector(tcx: TyCtxt<'_>, span: Span, def_id: LocalDefId, scalab
             // bools
             match element_ty.kind() {
                 ty::Int(_) | ty::Uint(_) | ty::Float(_) | ty::Bool => (),
+                // We need to treat a `bfloat` (`f16b`) as a primitive scalar
+                ty::Adt(def, _) if tcx.is_lang_item(def.did(), LangItem::F16B) => (),
                 _ => {
                     let mut err = tcx.dcx().struct_span_err(
                         span,
diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs
index 87b2bb94cd4f9..5c6aef8676716 100644
--- a/compiler/rustc_interface/src/util.rs
+++ b/compiler/rustc_interface/src/util.rs
@@ -74,6 +74,9 @@ pub(crate) fn add_configuration(
     if target_config.has_reliable_f16_math {
         cfg.insert((sym::target_has_reliable_f16_math, None));
     }
+    if target_config.has_reliable_f16b {
+        cfg.insert((sym::target_has_reliable_f16b, None));
+    }
     if target_config.has_reliable_f128 {
         cfg.insert((sym::target_has_reliable_f128, None));
     }
@@ -419,6 +422,7 @@ impl CodegenBackend for DummyCodegenBackend {
             internal_target_features,
             has_reliable_f16: true,
             has_reliable_f16_math: true,
+            has_reliable_f16b: true,
             has_reliable_f128: true,
             has_reliable_f128_math: true,
         }
diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs
index 22fbb306849da..8f2bb97e12a8f 100644
--- a/compiler/rustc_middle/src/ty/layout.rs
+++ b/compiler/rustc_middle/src/ty/layout.rs
@@ -137,6 +137,11 @@ impl abi::Float {
         use abi::Float::*;
         match *self {
             F16 => tcx.types.f16,
+            F16B => Ty::new_adt(
+                tcx,
+                tcx.adt_def(tcx.require_lang_item(LangItem::F16B, DUMMY_SP)),
+                ty::List::empty(),
+            ),
             F32 => tcx.types.f32,
             F64 => tcx.types.f64,
             F128 => tcx.types.f128,
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index 39d9d2b7b05ce..098b6595f166e 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -621,6 +621,7 @@ symbols! {
         cfg_target_has_atomic,
         cfg_target_has_atomic_equal_alignment,
         cfg_target_has_reliable_f16_f128,
+        cfg_target_has_reliable_f16b,
         cfg_target_has_threads,
         cfg_target_object_format,
         cfg_target_thread_local,
@@ -958,6 +959,7 @@ symbols! {
         external_doc,
         f16,
         f16_nan,
+        f16b,
         f16c_target_feature,
         f32,
         f32_nan,
@@ -2125,6 +2127,7 @@ symbols! {
         target_has_atomic_primitive_alignment,
         target_has_reliable_f16,
         target_has_reliable_f16_math,
+        target_has_reliable_f16b,
         target_has_reliable_f128,
         target_has_reliable_f128_math,
         target_has_threads,
diff --git a/compiler/rustc_ty_utils/src/layout.rs b/compiler/rustc_ty_utils/src/layout.rs
index fe093adcbf3f4..dee4b785e418d 100644
--- a/compiler/rustc_ty_utils/src/layout.rs
+++ b/compiler/rustc_ty_utils/src/layout.rs
@@ -11,6 +11,7 @@ use rustc_abi::{
 };
 use rustc_hashes::Hash64;
 use rustc_hir as hir;
+use rustc_hir::attrs::lang_items::LangItem;
 use rustc_hir::find_attr;
 use rustc_index::{Idx as _, IndexVec};
 use rustc_middle::query::Providers;
@@ -737,7 +738,7 @@ fn layout_of_uncached<'tcx>(
                         .is_sized(tcx, typing_env)
                 });
 
-            let layout = cx
+            let mut layout = cx
                 .calc
                 .layout_of_struct_or_enum(
                     &def.repr(),
@@ -804,6 +805,13 @@ fn layout_of_uncached<'tcx>(
                 }
             }
 
+            if tcx.is_lang_item(def.did(), LangItem::F16B) {
+                let bfloat = scalar_unit(Primitive::Float(abi::Float::F16B));
+                assert_eq!(layout.size, abi::Float::F16B.size());
+                layout.align = abi::Float::F16B.align(cx);
+                layout.backend_repr = BackendRepr::Scalar(bfloat);
+            }
+
             tcx.mk_layout(layout)
         }
 
diff --git a/library/core/Cargo.toml b/library/core/Cargo.toml
index 3f5f9f454a99d..d8f4ec0528dee 100644
--- a/library/core/Cargo.toml
+++ b/library/core/Cargo.toml
@@ -36,6 +36,7 @@ check-cfg = [
     # gate tests.
     'cfg(target_has_reliable_f16)',
     'cfg(target_has_reliable_f16_math)',
+    'cfg(target_has_reliable_f16b)',
     'cfg(target_has_reliable_f128)',
     'cfg(target_has_reliable_f128_math)',
     # Prevents use of a static variable for providing platform specific RawOsError
diff --git a/library/core/src/num/bfloat.rs b/library/core/src/num/bfloat.rs
new file mode 100644
index 0000000000000..4937dd8448c60
--- /dev/null
+++ b/library/core/src/num/bfloat.rs
@@ -0,0 +1,183 @@
+//! The 16-bit brain floating-point type.
+
+#![unstable(feature = "f16b", issue = "160630")]
+
+use crate::{fmt, mem};
+
+/// A 16-bit brain floating-point value.
+///
+/// This type stores values using the bfloat16 encoding. It deliberately
+/// exposes only raw-bit construction, comparison, formatting, and lossless
+/// widening to [`f32`].
+///
+/// The 16-bit brain floating-point intends to preserve the dynamic range of
+/// a 32-bit floating-point value while using half the storage. It does
+/// this by using 8 bits for the exponent, the same as `f32`, but only
+/// using 7 bits for the mantissa. See [Wikipedia on bfloat16][wikipedia] for
+/// more information.
+///
+/// [wikipedia]: https://en.wikipedia.org/wiki/Bfloat16_floating-point_format
+#[lang = "f16b"]
+#[doc(alias = "bf16")] // what hardware often names it
+#[doc(alias = "bfloat")] // LLVM's name
+#[doc(alias = "bfloat16")] // Wikipedia's name
+#[doc(alias = "bfloat16_t")] // The C++ `stdfloat` name
+#[allow(non_camel_case_types)]
+#[repr(transparent)]
+#[unstable(feature = "f16b", issue = "160630")]
+pub struct f16b(u16);
+
+#[doc(test(attr(
+    feature(cfg_target_has_reliable_f16b),
+    allow(internal_features, unused_features)
+)))]
+impl f16b {
+    /// Raw transmutation from `u16`.
+    ///
+    /// This is currently identical to `transmute::<u16, f16b>(v)` on all platforms.
+    /// It turns out this is incredibly portable, for two reasons:
+    ///
+    /// * Floats and Ints have the same endianness on all supported platforms.
+    /// * IEEE 754 very precisely specifies the bit layout of floats.
+    ///
+    /// However there is one caveat: prior to the 2008 version of IEEE 754, how
+    /// to interpret the NaN signaling bit wasn't actually specified. Most platforms
+    /// (notably x86 and ARM) picked the interpretation that was ultimately
+    /// standardized in 2008, but some didn't (notably MIPS). As a result, all
+    /// signaling NaNs on MIPS are quiet NaNs on x86, and vice-versa.
+    ///
+    /// Rather than trying to preserve signaling-ness cross-platform, this
+    /// implementation favors preserving the exact bits. This means that
+    /// any payloads encoded in NaNs will be preserved even if the result of
+    /// this method is sent over the network from an x86 machine to a MIPS one.
+    ///
+    /// If the results of this method are only manipulated by the same
+    /// architecture that produced them, then there is no portability concern.
+    ///
+    /// If the input isn't NaN, then there is no portability concern.
+    ///
+    /// If you don't care about signalingness (very likely), then there is no
+    /// portability concern.
+    ///
+    /// Note that this function is distinct from `as` casting, which attempts to
+    /// preserve the *numeric* value, and not the bitwise value.
+    ///
+    /// ```no_run
+    /// #![feature(f16b)]
+    /// # #[cfg(target_has_reliable_f16b)] {
+    /// use core::num::f16b;
+    ///
+    /// let v = f16b::from_bits(0x4148);
+    /// assert_eq!(f32::from(v), 12.5);
+    /// # }
+    /// ```
+    #[inline]
+    #[must_use]
+    #[unstable(feature = "f16b", issue = "160630")]
+    pub const fn from_bits(bits: u16) -> Self {
+        // SAFETY: `f16b` and `u16` have the same size, and every bit pattern is valid.
+        unsafe { mem::transmute(bits) }
+    }
+
+    /// Raw transmutation to `u16`.
+    ///
+    /// This is currently identical to `transmute::<f16b, u16>(self)` on all platforms.
+    ///
+    /// See [`from_bits`](#method.from_bits) for some discussion of the
+    /// portability of this operation (there are almost no issues).
+    ///
+    /// Note that this function is distinct from `as` casting, which attempts to
+    /// preserve the *numeric* value, and not the bitwise value.
+    ///
+    /// ```no_run
+    /// #![feature(f16b)]
+    /// # #[cfg(target_has_reliable_f16b)] {
+    /// use core::num::f16b;
+    ///
+    /// assert_eq!(f16b::from_bits(0x4148).to_bits(), 0x4148);
+    /// # }
+    /// ```
+    #[inline]
+    #[unstable(feature = "f16b", issue = "160630")]
+    #[must_use = "this returns the result of the operation, without modifying the original"]
+    pub const fn to_bits(self) -> u16 {
+        // SAFETY: `f16b` and `u16` have the same size, and every bit pattern is valid.
+        unsafe { mem::transmute(self) }
+    }
+}
+
+// FIXME(f16b) - This should be a `#[rustc_intrinsic]` using LLVM's `fpext`
+// with this implementation constituting the fallback.
+#[inline]
+const fn widen(value: f16b) -> f32 {
+    f32::from_bits((value.to_bits() as u32) << 16)
+}
+
+#[unstable(feature = "f16b", issue = "160630")]
+impl Copy for f16b {}
+
+#[unstable(feature = "f16b", issue = "160630")]
+impl Clone for f16b {
+    #[inline]
+    fn clone(&self) -> Self {
+        *self
+    }
+}
+
+#[unstable(feature = "f16b", issue = "160630")]
+impl Default for f16b {
+    #[inline]
+    fn default() -> Self {
+        Self::from_bits(0)
+    }
+}
+
+#[unstable(feature = "f16b", issue = "160630")]
+impl PartialEq for f16b {
+    #[inline]
+    fn eq(&self, other: &Self) -> bool {
+        widen(*self).eq(&widen(*other))
+    }
+}
+
+#[unstable(feature = "f16b", issue = "160630")]
+impl PartialOrd for f16b {
+    #[inline]
+    fn partial_cmp(&self, other: &Self) -> Option<crate::cmp::Ordering> {
+        widen(*self).partial_cmp(&widen(*other))
+    }
+}
+
+#[unstable(feature = "f16b", issue = "160630")]
+impl From<f16b> for f32 {
+    #[inline]
+    fn from(value: f16b) -> Self {
+        widen(value)
+    }
+}
+
+#[unstable(feature = "f16b", issue = "160630")]
+impl fmt::Debug for f16b {
+    #[inline]
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Debug::fmt(&widen(*self), formatter)
+    }
+}
+
+#[cfg(not(no_fp_fmt_parse))]
+#[unstable(feature = "f16b", issue = "160630")]
+impl fmt::LowerExp for f16b {
+    #[inline]
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::LowerExp::fmt(&widen(*self), formatter)
+    }
+}
+
+#[cfg(not(no_fp_fmt_parse))]
+#[unstable(feature = "f16b", issue = "160630")]
+impl fmt::UpperExp for f16b {
+    #[inline]
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::UpperExp::fmt(&widen(*self), formatter)
+    }
+}
diff --git a/library/core/src/num/mod.rs b/library/core/src/num/mod.rs
index db41d23770477..f5c7c853ef590 100644
--- a/library/core/src/num/mod.rs
+++ b/library/core/src/num/mod.rs
@@ -43,6 +43,7 @@ mod int_macros; // import int_impl!
 #[macro_use]
 mod uint_macros; // import uint_impl!
 
+mod bfloat;
 mod complex;
 mod error;
 #[cfg(not(no_fp_fmt_parse))]
@@ -56,6 +57,8 @@ mod wrapping;
 #[doc(hidden)]
 pub mod niche_types;
 
+#[unstable(feature = "f16b", issue = "160630")]
+pub use bfloat::f16b;
 #[unstable(feature = "complex_numbers", issue = "154023")]
 pub use complex::Complex;
 #[stable(feature = "int_error_matching", since = "1.55.0")]
diff --git a/src/doc/unstable-book/src/language-features/f16b.md b/src/doc/unstable-book/src/language-features/f16b.md
new file mode 100644
index 0000000000000..c18289921e032
--- /dev/null
+++ b/src/doc/unstable-book/src/language-features/f16b.md
@@ -0,0 +1,11 @@
+# `f16b`
+
+The tracking issue for this feature is: [#160630]
+
+[#160630]: https://github.com/rust-lang/rust/issues/160630
+
+---
+
+Enable the `core::num::f16b` type for values stored in the bfloat16 format.
+
+`f16b` is a nominal library type, not a primitive floating-point type: it has no literal suffix, arithmetic operators, or numeric `as` casts. A compiler lang item connects its layout to the chosen backend's representation.

From 47abb2aab2d224c4c415da238d32b8906af0e8c5 Mon Sep 17 00:00:00 2001
From: James Barford-Evans <james.barford-evans@arm.com>
Date: Fri, 18 Sep 2026 10:34:03 +0100
Subject: [PATCH 15/28] Add `F16B` to `Float` enum & wireup trivial matches

---
 compiler/rustc_abi/src/layout/ty.rs                      | 2 +-
 compiler/rustc_abi/src/lib.rs                            | 8 +++++++-
 compiler/rustc_codegen_llvm/src/va_arg.rs                | 6 +++++-
 compiler/rustc_codegen_ssa/src/mir/naked_asm.rs          | 3 +++
 compiler/rustc_codegen_ssa/src/traits/type_.rs           | 2 ++
 compiler/rustc_public/src/abi.rs                         | 3 ++-
 compiler/rustc_public/src/unstable/convert/stable/abi.rs | 1 +
 compiler/rustc_target/src/callconv/mips64.rs             | 1 +
 compiler/rustc_target/src/callconv/sparc64.rs            | 1 +
 9 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/compiler/rustc_abi/src/layout/ty.rs b/compiler/rustc_abi/src/layout/ty.rs
index 5033f887a60f9..e51255dc5963f 100644
--- a/compiler/rustc_abi/src/layout/ty.rs
+++ b/compiler/rustc_abi/src/layout/ty.rs
@@ -342,7 +342,7 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
             Primitive::Float(float @ (Float::F16 | Float::F32 | Float::F64 | Float::F128)) => {
                 Some(Numeric::Float(float))
             }
-            Primitive::Pointer(..) => None,
+            Primitive::Pointer(..) | Primitive::Float(Float::F16B) => None,
         }
     }
 
diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs
index b056fdc73d40b..f10f9990debb4 100644
--- a/compiler/rustc_abi/src/lib.rs
+++ b/compiler/rustc_abi/src/lib.rs
@@ -1420,6 +1420,10 @@ impl Integer {
 #[cfg_attr(feature = "nightly", derive(StableHash))]
 pub enum Float {
     F16,
+    /// `f16b`. This is not a builtin type in Rust (it is exposed as a lang item),
+    /// but it is a builtin type in LLVM so needs to be explicitly represented
+    /// in the backend.
+    F16B,
     F32,
     F64,
     F128,
@@ -1431,6 +1435,7 @@ impl Float {
 
         match self {
             F16 => Size::from_bits(16),
+            F16B => Size::from_bits(16),
             F32 => Size::from_bits(32),
             F64 => Size::from_bits(64),
             F128 => Size::from_bits(128),
@@ -1442,7 +1447,7 @@ impl Float {
         let dl = cx.data_layout();
 
         AbiAlign::new(match self {
-            F16 => dl.f16_align,
+            F16 | F16B => dl.f16_align,
             F32 => dl.f32_align,
             F64 => dl.f64_align,
             F128 => dl.f128_align,
@@ -1454,6 +1459,7 @@ impl Float {
 
         match self {
             F16 => "f16",
+            F16B => "f16b",
             F32 => "f32",
             F64 => "f64",
             F128 => "f128",
diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs
index ca57bee3e6271..e7883871f57e8 100644
--- a/compiler/rustc_codegen_llvm/src/va_arg.rs
+++ b/compiler/rustc_codegen_llvm/src/va_arg.rs
@@ -95,7 +95,7 @@ fn get_param_type_alignment<'ll, 'tcx>(
                 Integer::I128 => return Align::EIGHT,
             },
             Primitive::Float(float) => match float {
-                Float::F16 | Float::F32 => unreachable!(),
+                Float::F16 | Float::F16B | Float::F32 => unreachable!(),
                 Float::F64 => { /* fall through */ }
                 Float::F128 => return Align::from_bytes(16).unwrap(),
             },
@@ -481,7 +481,11 @@ fn emit_s390x_va_arg<'ll, 'tcx>(
                 Primitive::Float(Float::F16 | Float::F32 | Float::F64) => true,
                 Primitive::Float(Float::F128) => false,
                 Primitive::Int(_, _) | Primitive::Pointer(_) => false,
+                Primitive::Float(Float::F16B) => {
+                    bug!("`f16b` use in varadics unsupported on s390x")
+                }
             },
+
             _ => false,
         }
     };
diff --git a/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs b/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs
index ea58b1cd3c53c..3162463d222ee 100644
--- a/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/naked_asm.rs
@@ -508,6 +508,9 @@ fn wasm_primitive(primitive: Primitive, ptr_type: &'static str) -> &'static str
             Integer::I128 => "i64, i64",
         },
         Primitive::Float(float) => match float {
+            // This could probably use an f32 for WASM however has not been
+            // verified so leaving as a `bug!(...)` for now.
+            Float::F16B => bug!("`f16b` unsupported on wasm"),
             Float::F16 | Float::F32 => "f32",
             Float::F64 => "f64",
             Float::F128 => "i64, i64",
diff --git a/compiler/rustc_codegen_ssa/src/traits/type_.rs b/compiler/rustc_codegen_ssa/src/traits/type_.rs
index 1b70d38dc9e85..986481e7f132f 100644
--- a/compiler/rustc_codegen_ssa/src/traits/type_.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/type_.rs
@@ -18,6 +18,7 @@ pub trait BaseTypeCodegenMethods: BackendTypes {
     fn type_isize(&self) -> Self::Type;
 
     fn type_f16(&self) -> Self::Type;
+    fn type_f16b(&self) -> Self::Type;
     fn type_f32(&self) -> Self::Type;
     fn type_f64(&self) -> Self::Type;
     fn type_f128(&self) -> Self::Type;
@@ -67,6 +68,7 @@ pub trait DerivedTypeCodegenMethods<'tcx>:
         use Float::*;
         match f {
             F16 => self.type_f16(),
+            F16B => self.type_f16b(),
             F32 => self.type_f32(),
             F64 => self.type_f64(),
             F128 => self.type_f128(),
diff --git a/compiler/rustc_public/src/abi.rs b/compiler/rustc_public/src/abi.rs
index b760ed98c7111..67d609c780c42 100644
--- a/compiler/rustc_public/src/abi.rs
+++ b/compiler/rustc_public/src/abi.rs
@@ -496,6 +496,7 @@ pub enum IntegerLength {
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Serialize)]
 pub enum FloatLength {
     F16,
+    F16B,
     F32,
     F64,
     F128,
@@ -516,7 +517,7 @@ impl IntegerLength {
 impl FloatLength {
     pub fn bits(self) -> usize {
         match self {
-            FloatLength::F16 => 16,
+            FloatLength::F16 | FloatLength::F16B => 16,
             FloatLength::F32 => 32,
             FloatLength::F64 => 64,
             FloatLength::F128 => 128,
diff --git a/compiler/rustc_public/src/unstable/convert/stable/abi.rs b/compiler/rustc_public/src/unstable/convert/stable/abi.rs
index 766c522958db7..3c268a6dd23a4 100644
--- a/compiler/rustc_public/src/unstable/convert/stable/abi.rs
+++ b/compiler/rustc_public/src/unstable/convert/stable/abi.rs
@@ -463,6 +463,7 @@ impl<'tcx> Stable<'tcx> for rustc_abi::Float {
     fn stable(&self, _: &mut Tables<'_, BridgeTys>, _: &CompilerCtxt<'_, BridgeTys>) -> Self::T {
         match self {
             rustc_abi::Float::F16 => FloatLength::F16,
+            rustc_abi::Float::F16B => FloatLength::F16B,
             rustc_abi::Float::F32 => FloatLength::F32,
             rustc_abi::Float::F64 => FloatLength::F64,
             rustc_abi::Float::F128 => FloatLength::F128,
diff --git a/compiler/rustc_target/src/callconv/mips64.rs b/compiler/rustc_target/src/callconv/mips64.rs
index 8002f98507ba8..0e3ccf33c3ffe 100644
--- a/compiler/rustc_target/src/callconv/mips64.rs
+++ b/compiler/rustc_target/src/callconv/mips64.rs
@@ -31,6 +31,7 @@ where
                 match float {
                     // C does not have the f16 type
                     Float::F16 => None,
+                    Float::F16B => unreachable!("`f16b` unsupported on mips64"),
                     Float::F32 => Some(Reg::f32()),
                     Float::F64 => Some(Reg::f64()),
                     Float::F128 => Some(Reg::f128()),
diff --git a/compiler/rustc_target/src/callconv/sparc64.rs b/compiler/rustc_target/src/callconv/sparc64.rs
index bfaa3f3cb19c8..abef388ef2e12 100644
--- a/compiler/rustc_target/src/callconv/sparc64.rs
+++ b/compiler/rustc_target/src/callconv/sparc64.rs
@@ -58,6 +58,7 @@ fn classify<'a, Ty, C>(
                         Float::F16 => {
                             // Match LLVM by passing `f16` in integer registers.
                         }
+                        Float::F16B => unreachable!("`f16b` unsupported on sparc64"),
                     }
                 } else {
                     /* pass unaligned floats in integer registers */

From 3e5d69e7a9caec7260136a0cf6cf8e5eb352652c Mon Sep 17 00:00:00 2001
From: James Barford-Evans <james.barford-evans@arm.com>
Date: Fri, 18 Sep 2026 10:34:03 +0100
Subject: [PATCH 16/28] Wire up f16b in backends

---
 compiler/rustc_codegen_cranelift/src/common.rs   |  1 +
 compiler/rustc_codegen_cranelift/src/lib.rs      |  1 +
 compiler/rustc_codegen_gcc/src/base.rs           |  2 ++
 compiler/rustc_codegen_gcc/src/context.rs        |  3 +++
 compiler/rustc_codegen_gcc/src/lib.rs            |  2 ++
 compiler/rustc_codegen_gcc/src/type_.rs          |  8 ++++++++
 compiler/rustc_codegen_llvm/src/abi.rs           |  1 +
 compiler/rustc_codegen_llvm/src/intrinsic.rs     |  3 +++
 compiler/rustc_codegen_llvm/src/llvm_util.rs     | 14 ++++++++++++++
 compiler/rustc_codegen_llvm/src/type_.rs         |  4 ++++
 compiler/rustc_codegen_ssa/src/lib.rs            |  3 +++
 compiler/rustc_codegen_ssa/src/traits/backend.rs |  1 +
 compiler/rustc_session/src/config/cfg.rs         |  1 +
 src/tools/miri/src/bin/miri.rs                   |  1 +
 14 files changed, 45 insertions(+)

diff --git a/compiler/rustc_codegen_cranelift/src/common.rs b/compiler/rustc_codegen_cranelift/src/common.rs
index d31c8fc810b2c..30bd2f28af53f 100644
--- a/compiler/rustc_codegen_cranelift/src/common.rs
+++ b/compiler/rustc_codegen_cranelift/src/common.rs
@@ -35,6 +35,7 @@ pub(crate) fn scalar_to_clif_type(tcx: TyCtxt<'_>, scalar: Scalar) -> Type {
         },
         Primitive::Float(float) => match float {
             Float::F16 => types::F16,
+            Float::F16B => bug!("f16b is not supported by the Cranelift codegen backend"),
             Float::F32 => types::F32,
             Float::F64 => types::F64,
             Float::F128 => types::F128,
diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs
index d4773e8199aed..6629cc834bbd6 100644
--- a/compiler/rustc_codegen_cranelift/src/lib.rs
+++ b/compiler/rustc_codegen_cranelift/src/lib.rs
@@ -196,6 +196,7 @@ impl CodegenBackend for CraneliftCodegenBackend {
             // available in Cranelift.
             has_reliable_f16: has_reliable_f16_f128,
             has_reliable_f16_math: has_reliable_f16_f128,
+            has_reliable_f16b: false,
             has_reliable_f128: has_reliable_f16_f128,
             has_reliable_f128_math,
         }
diff --git a/compiler/rustc_codegen_gcc/src/base.rs b/compiler/rustc_codegen_gcc/src/base.rs
index 101af0bb0bff1..498ea4b45a8f8 100644
--- a/compiler/rustc_codegen_gcc/src/base.rs
+++ b/compiler/rustc_codegen_gcc/src/base.rs
@@ -214,6 +214,7 @@ pub fn compile_codegen_unit(
             // -fsyntax-only), forbid the compilation when get_target_info() is called on a
             // context.
             let f16_type_supported = target_info.supports_target_dependent_type(CType::Float16);
+            let f16b_type_supported = target_info.supports_target_dependent_type(CType::BFloat16);
             let f32_type_supported = target_info.supports_target_dependent_type(CType::Float32);
             let f64_type_supported = target_info.supports_target_dependent_type(CType::Float64);
             let f128_type_supported = target_info.supports_target_dependent_type(CType::Float128);
@@ -225,6 +226,7 @@ pub fn compile_codegen_unit(
                 tcx,
                 u128_type_supported,
                 f16_type_supported,
+                f16b_type_supported,
                 f32_type_supported,
                 f64_type_supported,
                 f128_type_supported,
diff --git a/compiler/rustc_codegen_gcc/src/context.rs b/compiler/rustc_codegen_gcc/src/context.rs
index 38e0e5f329f76..a41255be9e1ea 100644
--- a/compiler/rustc_codegen_gcc/src/context.rs
+++ b/compiler/rustc_codegen_gcc/src/context.rs
@@ -72,6 +72,7 @@ pub struct CodegenCx<'gcc, 'tcx> {
 
     pub supports_128bit_integers: bool,
     pub supports_f16_type: bool,
+    pub supports_f16b_type: bool,
     pub supports_f32_type: bool,
     pub supports_f64_type: bool,
     pub supports_f128_type: bool,
@@ -140,6 +141,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
         tcx: TyCtxt<'tcx>,
         supports_128bit_integers: bool,
         supports_f16_type: bool,
+        supports_f16b_type: bool,
         supports_f32_type: bool,
         supports_f64_type: bool,
         supports_f128_type: bool,
@@ -276,6 +278,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
 
             supports_128bit_integers,
             supports_f16_type,
+            supports_f16b_type,
             supports_f32_type,
             supports_f64_type,
             supports_f128_type,
diff --git a/compiler/rustc_codegen_gcc/src/lib.rs b/compiler/rustc_codegen_gcc/src/lib.rs
index d7a3ef3b4a6c5..79802a670a3f0 100644
--- a/compiler/rustc_codegen_gcc/src/lib.rs
+++ b/compiler/rustc_codegen_gcc/src/lib.rs
@@ -503,6 +503,7 @@ fn target_config(sess: &EarlySession, target_info: &SharedTargetInfo) -> TargetC
     );
 
     let has_reliable_f16 = target_info.supports_target_dependent_type(CType::Float16);
+    let has_reliable_f16b = target_info.supports_target_dependent_type(CType::BFloat16);
     let has_reliable_f128 = target_info.supports_target_dependent_type(CType::Float128);
 
     TargetConfig {
@@ -510,6 +511,7 @@ fn target_config(sess: &EarlySession, target_info: &SharedTargetInfo) -> TargetC
         // There are no known bugs with GCC support for f16 or f128
         has_reliable_f16,
         has_reliable_f16_math: has_reliable_f16,
+        has_reliable_f16b,
         has_reliable_f128,
         has_reliable_f128_math: has_reliable_f128,
     }
diff --git a/compiler/rustc_codegen_gcc/src/type_.rs b/compiler/rustc_codegen_gcc/src/type_.rs
index 27b0d2079e63e..1d8582fe337ad 100644
--- a/compiler/rustc_codegen_gcc/src/type_.rs
+++ b/compiler/rustc_codegen_gcc/src/type_.rs
@@ -157,6 +157,14 @@ impl<'gcc, 'tcx> BaseTypeCodegenMethods for CodegenCx<'gcc, 'tcx> {
         bug!("unsupported float width 16")
     }
 
+    fn type_f16b(&self) -> Type<'gcc> {
+        #[cfg(feature = "master")]
+        if self.supports_f16b_type {
+            return self.context.new_c_type(CType::BFloat16);
+        }
+        bug!("unsupported type bfloat16")
+    }
+
     fn type_f32(&self) -> Type<'gcc> {
         #[cfg(feature = "master")]
         if self.supports_f32_type {
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs
index a45138849e4e0..dce8db6841b7f 100644
--- a/compiler/rustc_codegen_llvm/src/abi.rs
+++ b/compiler/rustc_codegen_llvm/src/abi.rs
@@ -159,6 +159,7 @@ impl LlvmType for Reg {
                     },
                     Primitive::Float(float) => match float {
                         Float::F16 => cx.type_f16(),
+                        Float::F16B => cx.type_f16b(),
                         Float::F32 => cx.type_f32(),
                         Float::F64 => cx.type_f64(),
                         Float::F128 => cx.type_f128(),
diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs
index db896fa9c0f2b..9a898429d0384 100644
--- a/compiler/rustc_codegen_llvm/src/intrinsic.rs
+++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs
@@ -337,6 +337,9 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     Primitive::Float(Float::F16) => {
                         bug!("the va_arg intrinsic does not support `f16`")
                     }
+                    Primitive::Float(Float::F16B) => {
+                        bug!("the va_arg intrinsic does not support `f16b`")
+                    }
                     Primitive::Float(Float::F32) => {
                         // c_double is actually f32 on avr.
                         if self.cx().sess().target.arch != Arch::Avr {
diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs
index 90f31e0598f2d..a790c75ae4d10 100644
--- a/compiler/rustc_codegen_llvm/src/llvm_util.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs
@@ -382,6 +382,7 @@ pub(crate) fn target_config(sess: &EarlySession) -> TargetConfig {
         internal_target_features,
         has_reliable_f16: true,
         has_reliable_f16_math: true,
+        has_reliable_f16b: true,
         has_reliable_f128: true,
         has_reliable_f128_math: true,
     };
@@ -419,6 +420,19 @@ fn update_target_reliable_float_cfg(target: &Target, cfg: &mut TargetConfig) {
         _ => true,
     };
 
+    // The heuristic for evaluating to true is twofold, namely;
+    //
+    // 1. Can LLVM compile an IR snippet containing `fpext bfloat %<var> to float`
+    // 2. Does the documentation indicate `bf16` support, can be seen in the
+    //    tracking issue; <https://github.com/rust-lang/rust/issues/160630>
+    cfg.has_reliable_f16b = match (target_arch, target_os) {
+        // This is similar to <https://github.com/llvm/llvm-project/issues/94434>, however
+        // does not work until LLVM 23 on Windows.
+        (Arch::Arm64EC, _) => major >= 23,
+        (Arch::AArch64 | Arch::X86_64 | Arch::RiscV64 | Arch::LoongArch64, _) => true,
+        _ => false,
+    };
+
     cfg.has_reliable_f128 = match (target_arch, target_os) {
         // Unsupported https://github.com/llvm/llvm-project/issues/121122
         (Arch::AmdGpu, _) => false,
diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs
index 881b8e3a8bd0e..ff1b5851db0f9 100644
--- a/compiler/rustc_codegen_llvm/src/type_.rs
+++ b/compiler/rustc_codegen_llvm/src/type_.rs
@@ -214,6 +214,10 @@ impl<'ll, CX: Borrow<SCx<'ll>>> BaseTypeCodegenMethods for GenericCx<'ll, CX> {
         unsafe { llvm::LLVMHalfTypeInContext(self.llcx()) }
     }
 
+    fn type_f16b(&self) -> &'ll Type {
+        unsafe { llvm::LLVMBFloatTypeInContext(self.llcx()) }
+    }
+
     fn type_f32(&self) -> &'ll Type {
         unsafe { llvm::LLVMFloatTypeInContext(self.llcx()) }
     }
diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs
index 1272b26ca0612..7a29ca383b24b 100644
--- a/compiler/rustc_codegen_ssa/src/lib.rs
+++ b/compiler/rustc_codegen_ssa/src/lib.rs
@@ -315,6 +315,9 @@ pub struct TargetConfig {
     pub has_reliable_f16: bool,
     /// Option for `cfg(target_has_reliable_f16_math)`, true if `f16` math calls work.
     pub has_reliable_f16_math: bool,
+    /// Option for `cfg(target_has_reliable_f16b)`, presently true if both the ABI
+    /// and LLVM version supports `f16b`.
+    pub has_reliable_f16b: bool,
     /// Option for `cfg(target_has_reliable_f128)`, true if `f128` basic arithmetic works.
     pub has_reliable_f128: bool,
     /// Option for `cfg(target_has_reliable_f128_math)`, true if `f128` math calls work.
diff --git a/compiler/rustc_codegen_ssa/src/traits/backend.rs b/compiler/rustc_codegen_ssa/src/traits/backend.rs
index 38d562d028f34..e06ac36fd758a 100644
--- a/compiler/rustc_codegen_ssa/src/traits/backend.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/backend.rs
@@ -52,6 +52,7 @@ pub trait CodegenBackend {
             // support the float types, rather than accidentally quietly skipping all tests.
             has_reliable_f16: true,
             has_reliable_f16_math: true,
+            has_reliable_f16b: true,
             has_reliable_f128: true,
             has_reliable_f128_math: true,
         }
diff --git a/compiler/rustc_session/src/config/cfg.rs b/compiler/rustc_session/src/config/cfg.rs
index a6decd5898689..05e3c346cd767 100644
--- a/compiler/rustc_session/src/config/cfg.rs
+++ b/compiler/rustc_session/src/config/cfg.rs
@@ -154,6 +154,7 @@ pub(crate) fn disallow_cfgs(sess: &Session, user_cfgs: &Cfg) {
             | (sym::target_has_atomic_load_store, Some(_))
             | (sym::target_has_reliable_f16, None | Some(_))
             | (sym::target_has_reliable_f16_math, None | Some(_))
+            | (sym::target_has_reliable_f16b, None | Some(_))
             | (sym::target_has_reliable_f128, None | Some(_))
             | (sym::target_has_reliable_f128_math, None | Some(_))
             | (sym::target_thread_local, None) => disallow(cfg, "--target"),
diff --git a/src/tools/miri/src/bin/miri.rs b/src/tools/miri/src/bin/miri.rs
index 62280f641e9fd..7b3f166b3bf87 100644
--- a/src/tools/miri/src/bin/miri.rs
+++ b/src/tools/miri/src/bin/miri.rs
@@ -221,6 +221,7 @@ impl CodegenBackend for MiriCodegenBackend {
             internal_target_features: native_target_config.internal_target_features,
 
             // The basic types and ABI always work.
+            has_reliable_f16b: true,
             has_reliable_f16: true,
             has_reliable_f128: true,
             // We always provide the f16 intrinsics, but some are provided via the host,

From 960fdef0a17852684dcf483598e70ecc81ad8498 Mon Sep 17 00:00:00 2001
From: James Barford-Evans <james.barford-evans@arm.com>
Date: Fri, 18 Sep 2026 10:34:03 +0100
Subject: [PATCH 17/28] Update Tidy rules to allow for `//@ revision`

---
 src/tools/tidy/src/style.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/tools/tidy/src/style.rs b/src/tools/tidy/src/style.rs
index 6066ad5805006..6d8a6371f89e3 100644
--- a/src/tools/tidy/src/style.rs
+++ b/src/tools/tidy/src/style.rs
@@ -74,6 +74,7 @@ const ANNOTATIONS_TO_IGNORE: &[&str] = &[
     "//@ lldb",
     "//@ cdb",
     "//@ normalize-stderr",
+    "//@ revisions",
 ];
 
 fn generate_problems<'a>(

From c1287ce657d3743bb19c3b1a91d93a941e358902 Mon Sep 17 00:00:00 2001
From: James Barford-Evans <james.barford-evans@arm.com>
Date: Fri, 18 Sep 2026 10:34:04 +0100
Subject: [PATCH 18/28] Update and write tests

---
 tests/assembly-llvm/f16b.rs                   | 124 ++++++++++++++++++
 tests/auxiliary/minicore.rs                   |  43 +++++-
 tests/codegen-llvm/float/f16b.rs              |  40 ++++++
 .../scalable-vectors/bf16-intrinsic.rs        |  34 +++++
 ...ature-gate-cfg-target-has-reliable-f16b.rs |   6 +
 ...e-gate-cfg-target-has-reliable-f16b.stderr |  12 ++
 tests/ui/feature-gates/feature-gate-f16b.rs   |  30 +++++
 .../ui/feature-gates/feature-gate-f16b.stderr | 121 +++++++++++++++++
 tests/ui/float/f16b-restrictions.rs           |  20 +++
 tests/ui/float/f16b-restrictions.stderr       |  43 ++++++
 tests/ui/float/f16b.rs                        |  47 +++++++
 tests/ui/parser/f16b.rs                       |  14 ++
 tests/ui/parser/f16b.stderr                   |  10 ++
 13 files changed, 542 insertions(+), 2 deletions(-)
 create mode 100644 tests/assembly-llvm/f16b.rs
 create mode 100644 tests/codegen-llvm/float/f16b.rs
 create mode 100644 tests/codegen-llvm/scalable-vectors/bf16-intrinsic.rs
 create mode 100644 tests/ui/feature-gates/feature-gate-cfg-target-has-reliable-f16b.rs
 create mode 100644 tests/ui/feature-gates/feature-gate-cfg-target-has-reliable-f16b.stderr
 create mode 100644 tests/ui/feature-gates/feature-gate-f16b.rs
 create mode 100644 tests/ui/feature-gates/feature-gate-f16b.stderr
 create mode 100644 tests/ui/float/f16b-restrictions.rs
 create mode 100644 tests/ui/float/f16b-restrictions.stderr
 create mode 100644 tests/ui/float/f16b.rs
 create mode 100644 tests/ui/parser/f16b.rs
 create mode 100644 tests/ui/parser/f16b.stderr

diff --git a/tests/assembly-llvm/f16b.rs b/tests/assembly-llvm/f16b.rs
new file mode 100644
index 0000000000000..373ebd0ac566d
--- /dev/null
+++ b/tests/assembly-llvm/f16b.rs
@@ -0,0 +1,124 @@
+//@ add-minicore
+//@ assembly-output: emit-asm
+//
+//@ revisions: AARCH64_LINUX AARCH64_BE AARCH64_DARWIN AARCH64_MSVC ARM64EC_MSVC X64_LINUX X64_WINDOWS_GNU X64_WINDOWS_MSVC RISCV64 LOONGARCH64
+//@[AARCH64_LINUX] compile-flags: -Copt-level=3 --target aarch64-unknown-linux-gnu
+//@[AARCH64_LINUX] needs-llvm-components: aarch64
+//@[AARCH64_LINUX] filecheck-flags: --check-prefixes AARCH64,AARCH64-NOTAPPLE
+//@[AARCH64_BE] compile-flags: -Copt-level=3 --target aarch64_be-unknown-linux-gnu
+//@[AARCH64_BE] needs-llvm-components: aarch64
+//@[AARCH64_BE] filecheck-flags: --check-prefixes AARCH64,AARCH64-NOTAPPLE
+//@[AARCH64_DARWIN] compile-flags: -Copt-level=3 --target aarch64-apple-darwin
+//@[AARCH64_DARWIN] needs-llvm-components: aarch64
+//@[AARCH64_DARWIN] filecheck-flags: --check-prefixes AARCH64,AARCH64-APPLE
+//@[AARCH64_MSVC] compile-flags: -Copt-level=3 --target aarch64-pc-windows-msvc
+//@[AARCH64_MSVC] needs-llvm-components: aarch64
+//@[AARCH64_MSVC] filecheck-flags: --check-prefixes AARCH64,AARCH64-NOTAPPLE
+//@[ARM64EC_MSVC] compile-flags: -Copt-level=3 --target arm64ec-pc-windows-msvc
+//@[ARM64EC_MSVC] needs-llvm-components: aarch64
+//@[ARM64EC_MSVC] min-llvm-version: 23
+//@[ARM64EC_MSVC] filecheck-flags: --check-prefixes AARCH64,AARCH64-NOTAPPLE
+//@[X64_LINUX] compile-flags: -Copt-level=3 -Cllvm-args=-x86-asm-syntax=intel --target x86_64-unknown-linux-gnu
+//@[X64_LINUX] needs-llvm-components: x86
+//@[X64_LINUX] filecheck-flags: --check-prefixes X64,X64-LINUX
+//@[X64_WINDOWS_GNU] compile-flags: -Copt-level=3 -Cllvm-args=-x86-asm-syntax=intel --target x86_64-pc-windows-gnu
+//@[X64_WINDOWS_GNU] needs-llvm-components: x86
+//@[X64_WINDOWS_GNU] filecheck-flags: --check-prefixes X64,X64-WINDOWS
+//@[X64_WINDOWS_MSVC] compile-flags: -Copt-level=3 -Cllvm-args=-x86-asm-syntax=intel --target x86_64-pc-windows-msvc
+//@[X64_WINDOWS_MSVC] needs-llvm-components: x86
+//@[X64_WINDOWS_MSVC] filecheck-flags: --check-prefixes X64,X64-WINDOWS
+//@[RISCV64] compile-flags: -Copt-level=3 --target riscv64gc-unknown-linux-gnu
+//@[RISCV64] needs-llvm-components: riscv
+//@[LOONGARCH64] compile-flags: -Copt-level=3 --target loongarch64-unknown-linux-gnu
+//@[LOONGARCH64] needs-llvm-components: loongarch
+
+#![feature(f16b, no_core)]
+#![no_core]
+#![crate_type = "lib"]
+#![allow(improper_ctypes_definitions)]
+
+// Check that the assembly that rustc generates matches what clang emits.
+
+extern crate minicore;
+
+use minicore::From;
+use minicore::num::f16b;
+
+// CHECK-LABEL: identity_f16b
+// AARCH64: ret
+// X64: ret
+// RISCV64: fmv.x.w a0, fa0
+// RISCV64-NEXT: lui a1, 1048560
+// RISCV64-NEXT: or a0, a0, a1
+// RISCV64-NEXT: fmv.w.x fa0, a0
+// RISCV64-NEXT: ret
+// LOONGARCH64: movfr2gr.s $a0, $fa0
+// LOONGARCH64-NEXT: lu12i.w $a1, -16
+// LOONGARCH64-NEXT: or $a0, $a0, $a1
+// LOONGARCH64-NEXT: movgr2fr.w $fa0, $a0
+// LOONGARCH64-NEXT: ret
+#[unsafe(no_mangle)]
+pub extern "C" fn identity_f16b(value: f16b) -> f16b {
+    value
+}
+
+// CHECK-LABEL: f16b_to_bits
+// AARCH64-NOTAPPLE: fmov w0, s0
+// AARCH64-APPLE: fmov w8, s0
+// AARCH64-APPLE-NEXT: and w0, w8, #0xffff
+// AARCH64-NEXT: ret
+// X64: pextrw eax, xmm0, 0
+// X64-NEXT: ret
+// RISCV64: fmv.x.w a0, fa0
+// RISCV64-NEXT: slli a0, a0, 48
+// RISCV64-NEXT: srli a0, a0, 48
+// RISCV64-NEXT: ret
+// LOONGARCH64: movfr2gr.s $a0, $fa0
+// LOONGARCH64-NEXT: bstrpick.d $a0, $a0, 15, 0
+// LOONGARCH64-NEXT: ret
+#[unsafe(no_mangle)]
+pub extern "C" fn f16b_to_bits(value: f16b) -> u16 {
+    value.to_bits()
+}
+
+// CHECK-LABEL: f16b_from_bits
+// AARCH64: fmov s0, w0
+// AARCH64-NEXT: ret
+// X64-LINUX: pinsrw xmm0, edi, 0
+// X64-LINUX-NEXT: ret
+// X64-WINDOWS: pinsrw xmm0, ecx, 0
+// X64-WINDOWS-NEXT: ret
+// RISCV64: lui a1, 1048560
+// RISCV64-NEXT: or a0, a0, a1
+// RISCV64-NEXT: fmv.w.x fa0, a0
+// RISCV64-NEXT: ret
+// LOONGARCH64: lu12i.w $a1, -16
+// LOONGARCH64-NEXT: or $a0, $a0, $a1
+// LOONGARCH64-NEXT: movgr2fr.w $fa0, $a0
+// LOONGARCH64-NEXT: ret
+#[unsafe(no_mangle)]
+pub extern "C" fn f16b_from_bits(bits: u16) -> f16b {
+    f16b::from_bits(bits)
+}
+
+// CHECK-LABEL: widen_f16b
+// AARCH64: fmov w8, s0
+// AARCH64-NEXT: lsl w8, w8, #16
+// AARCH64-NEXT: fmov s0, w8
+// AARCH64-NEXT: ret
+// X64: pextrw eax, xmm0, 0
+// X64-NEXT: shl eax, 16
+// X64-NEXT: movd xmm0, eax
+// X64-NEXT: ret
+// RISCV64: fmv.x.w a0, fa0
+// RISCV64-NEXT: slli a0, a0, 16
+// RISCV64-NEXT: fmv.w.x fa0, a0
+// RISCV64-NEXT: ret
+// LOONGARCH64: movfr2gr.s $a0, $fa0
+// LOONGARCH64-NEXT: slli.d $a0, $a0, 16
+// LOONGARCH64-NEXT: movgr2fr.w $fa0, $a0
+// LOONGARCH64-NEXT: ret
+#[unsafe(no_mangle)]
+pub extern "C" fn widen_f16b(value: f16b) -> f32 {
+    f32::from(value)
+}
diff --git a/tests/auxiliary/minicore.rs b/tests/auxiliary/minicore.rs
index 2b7eb0b9afbcf..f0ccc341a567d 100644
--- a/tests/auxiliary/minicore.rs
+++ b/tests/auxiliary/minicore.rs
@@ -29,6 +29,8 @@
     rustc_attrs,
     decl_macro,
     f16,
+    f16b,
+    cfg_target_has_reliable_f16b,
     f128,
     repr_simd,
     transparent_unions,
@@ -80,6 +82,10 @@ impl<T: PointeeSized> LegacyReceiver for &mut T {}
 #[lang = "copy"]
 pub trait Copy: Sized {}
 
+pub trait From<T>: Sized {
+    fn from(value: T) -> Self;
+}
+
 #[lang = "bikeshed_guaranteed_no_drop"]
 pub trait BikeshedGuaranteedNoDrop {}
 
@@ -360,7 +366,7 @@ pub const unsafe fn copy_nonoverlapping<T>(src: *const T, dst: *mut T, count: us
 pub mod mem {
     #[rustc_nounwind]
     #[rustc_intrinsic]
-    pub unsafe fn transmute<Src, Dst>(src: Src) -> Dst;
+    pub const unsafe fn transmute<Src, Dst>(src: Src) -> Dst;
 
     #[rustc_nounwind]
     #[rustc_intrinsic]
@@ -392,7 +398,40 @@ pub mod hint {
 }
 
 pub mod num {
-    use super::Copy;
+    use super::{Copy, From, mem};
+
+    #[rustc_intrinsic]
+    const unsafe fn unchecked_shl<T: Copy, U: Copy>(value: T, shift: U) -> T;
+
+    #[cfg(target_has_reliable_f16b)]
+    #[allow(non_camel_case_types)]
+    #[lang = "f16b"]
+    #[repr(transparent)]
+    pub struct f16b(u16);
+
+    #[cfg(target_has_reliable_f16b)]
+    impl f16b {
+        #[inline]
+        pub const fn from_bits(bits: u16) -> Self {
+            unsafe { mem::transmute(bits) }
+        }
+
+        #[inline]
+        pub const fn to_bits(self) -> u16 {
+            unsafe { mem::transmute(self) }
+        }
+    }
+
+    #[cfg(target_has_reliable_f16b)]
+    impl Copy for f16b {}
+
+    #[cfg(target_has_reliable_f16b)]
+    impl From<f16b> for f32 {
+        #[inline]
+        fn from(value: f16b) -> Self {
+            unsafe { mem::transmute(unchecked_shl(value.to_bits() as u32, 16u32)) }
+        }
+    }
 
     #[repr(C)]
     #[lang = "complex"]
diff --git a/tests/codegen-llvm/float/f16b.rs b/tests/codegen-llvm/float/f16b.rs
new file mode 100644
index 0000000000000..7de6a776190c9
--- /dev/null
+++ b/tests/codegen-llvm/float/f16b.rs
@@ -0,0 +1,40 @@
+//@ compile-flags: -Copt-level=3
+
+#![crate_type = "lib"]
+#![feature(f16b)]
+#![allow(improper_ctypes_definitions)]
+
+extern crate core;
+
+use core::num::f16b;
+
+// CHECK-LABEL: define{{.*}} bfloat @identity_f16b(bfloat
+#[no_mangle]
+pub extern "C" fn identity_f16b(value: f16b) -> f16b {
+    // CHECK: ret bfloat
+    value
+}
+
+// CHECK-LABEL: define{{.*}} i16 @f16b_to_bits(bfloat
+#[no_mangle]
+pub extern "C" fn f16b_to_bits(value: f16b) -> u16 {
+    // CHECK: bitcast bfloat %value to i16
+    value.to_bits()
+}
+
+// CHECK-LABEL: define{{.*}} bfloat @f16b_from_bits(i16
+#[no_mangle]
+pub extern "C" fn f16b_from_bits(bits: u16) -> f16b {
+    // CHECK: bitcast i16 %bits to bfloat
+    f16b::from_bits(bits)
+}
+
+// CHECK-LABEL: define{{.*}} float @widen_f16b(bfloat
+#[no_mangle]
+pub extern "C" fn widen_f16b(value: f16b) -> f32 {
+    // CHECK: bitcast bfloat %value to i16
+    // CHECK: zext i16
+    // CHECK: shl nuw i32 {{.*}}, 16
+    // CHECK: bitcast i32 {{.*}} to float
+    f32::from(value)
+}
diff --git a/tests/codegen-llvm/scalable-vectors/bf16-intrinsic.rs b/tests/codegen-llvm/scalable-vectors/bf16-intrinsic.rs
new file mode 100644
index 0000000000000..a080b36382389
--- /dev/null
+++ b/tests/codegen-llvm/scalable-vectors/bf16-intrinsic.rs
@@ -0,0 +1,34 @@
+//@ add-minicore
+//@ revisions: OPT-0 OPT-3
+//@[OPT-0] compile-flags: -Copt-level=0
+//@[OPT-3] compile-flags: -Copt-level=3
+//@ compile-flags: --target aarch64-unknown-linux-gnu
+//@ needs-llvm-components: aarch64
+
+#![crate_type = "lib"]
+#![feature(f16b, link_llvm_intrinsics, no_core, simd_ffi)]
+#![no_core]
+#![allow(non_camel_case_types)]
+
+extern crate minicore;
+
+use minicore::num::f16b;
+use minicore::simd::{Simd, f32x4};
+
+type bfloat16x8_t = Simd<f16b, 8>;
+
+#[unsafe(no_mangle)]
+#[target_feature(enable = "neon,bf16")]
+// CHECK-LABEL: define <4 x float> @vbfmmlaq_f32(
+// CHECK-SAME: <4 x float> %acc, <8 x bfloat> %lhs, <8 x bfloat> %rhs)
+pub unsafe extern "C" fn vbfmmlaq_f32(acc: f32x4, lhs: bfloat16x8_t, rhs: bfloat16x8_t) -> f32x4 {
+    unsafe extern "llvm-intrinsic" {
+        #[link_name = "llvm.aarch64.neon.bfmmla"]
+        fn bfmmla(acc: f32x4, lhs: bfloat16x8_t, rhs: bfloat16x8_t) -> f32x4;
+    }
+
+    // CHECK: [[RESULT:%.*]] = {{.*}}call <4 x float> @llvm.aarch64.neon.bfmmla(
+    // CHECK-SAME: <4 x float> %acc, <8 x bfloat> %lhs, <8 x bfloat> %rhs)
+    // CHECK: ret <4 x float> [[RESULT]]
+    unsafe { bfmmla(acc, lhs, rhs) }
+}
diff --git a/tests/ui/feature-gates/feature-gate-cfg-target-has-reliable-f16b.rs b/tests/ui/feature-gates/feature-gate-cfg-target-has-reliable-f16b.rs
new file mode 100644
index 0000000000000..a1c2f22782c4c
--- /dev/null
+++ b/tests/ui/feature-gates/feature-gate-cfg-target-has-reliable-f16b.rs
@@ -0,0 +1,6 @@
+//@ compile-flags: --check-cfg=cfg(target_has_reliable_f16b)
+
+fn main() {
+    cfg!(target_has_reliable_f16b);
+    //~^ ERROR `cfg(target_has_reliable_f16b)` is experimental and subject to change
+}
diff --git a/tests/ui/feature-gates/feature-gate-cfg-target-has-reliable-f16b.stderr b/tests/ui/feature-gates/feature-gate-cfg-target-has-reliable-f16b.stderr
new file mode 100644
index 0000000000000..46106f61ad3ae
--- /dev/null
+++ b/tests/ui/feature-gates/feature-gate-cfg-target-has-reliable-f16b.stderr
@@ -0,0 +1,12 @@
+error[E0658]: `cfg(target_has_reliable_f16b)` is experimental and subject to change
+  --> $DIR/feature-gate-cfg-target-has-reliable-f16b.rs:4:10
+   |
+LL |     cfg!(target_has_reliable_f16b);
+   |          ^^^^^^^^^^^^^^^^^^^^^^^^
+   |
+   = help: add `#![feature(cfg_target_has_reliable_f16b)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0658`.
diff --git a/tests/ui/feature-gates/feature-gate-f16b.rs b/tests/ui/feature-gates/feature-gate-f16b.rs
new file mode 100644
index 0000000000000..fc531c19ba5b1
--- /dev/null
+++ b/tests/ui/feature-gates/feature-gate-f16b.rs
@@ -0,0 +1,30 @@
+extern crate core;
+
+use core::num::f16b;
+//~^ ERROR use of unstable library feature `f16b`
+
+fn main() {
+    let _ = f16b::from_bits(0);
+    //~^ ERROR use of unstable library feature `f16b`
+    //~| ERROR use of unstable library feature `f16b`
+
+    let a = 0.0f16b;
+    //~^ ERROR  invalid suffix `f16b`
+
+    let _: f16b = 1.0;
+    //~^ ERROR use of unstable library feature `f16b`
+    //~| ERROR mismatched types
+
+    let x = f16b::from_bits(0x3f80);
+    //~^ ERROR use of unstable library feature `f16b`
+    //~| ERROR use of unstable library feature `f16b`
+    let _ = x + x;
+    //~^ ERROR cannot add `f16b` to `f16b`
+
+    let _ = 1u16 as f16b;
+    //~^ ERROR use of unstable library feature `f16b`
+    //~| ERROR non-primitive cast
+
+    let _ = x as f32;
+    //~^ ERROR non-primitive cast
+}
diff --git a/tests/ui/feature-gates/feature-gate-f16b.stderr b/tests/ui/feature-gates/feature-gate-f16b.stderr
new file mode 100644
index 0000000000000..3ba9299485276
--- /dev/null
+++ b/tests/ui/feature-gates/feature-gate-f16b.stderr
@@ -0,0 +1,121 @@
+error: invalid suffix `f16b` for float literal
+  --> $DIR/feature-gate-f16b.rs:11:13
+   |
+LL |     let a = 0.0f16b;
+   |             ^^^^^^^ invalid suffix `f16b`
+   |
+   = help: valid suffixes are `f32` and `f64`
+
+error[E0658]: use of unstable library feature `f16b`
+  --> $DIR/feature-gate-f16b.rs:3:5
+   |
+LL | use core::num::f16b;
+   |     ^^^^^^^^^^^^^^^
+   |
+   = note: see issue #160630 <https://github.com/rust-lang/rust/issues/160630> for more information
+   = help: add `#![feature(f16b)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
+
+error[E0658]: use of unstable library feature `f16b`
+  --> $DIR/feature-gate-f16b.rs:7:13
+   |
+LL |     let _ = f16b::from_bits(0);
+   |             ^^^^
+   |
+   = note: see issue #160630 <https://github.com/rust-lang/rust/issues/160630> for more information
+   = help: add `#![feature(f16b)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
+
+error[E0658]: use of unstable library feature `f16b`
+  --> $DIR/feature-gate-f16b.rs:14:12
+   |
+LL |     let _: f16b = 1.0;
+   |            ^^^^
+   |
+   = note: see issue #160630 <https://github.com/rust-lang/rust/issues/160630> for more information
+   = help: add `#![feature(f16b)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
+
+error[E0658]: use of unstable library feature `f16b`
+  --> $DIR/feature-gate-f16b.rs:18:13
+   |
+LL |     let x = f16b::from_bits(0x3f80);
+   |             ^^^^
+   |
+   = note: see issue #160630 <https://github.com/rust-lang/rust/issues/160630> for more information
+   = help: add `#![feature(f16b)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
+
+error[E0658]: use of unstable library feature `f16b`
+  --> $DIR/feature-gate-f16b.rs:24:21
+   |
+LL |     let _ = 1u16 as f16b;
+   |                     ^^^^
+   |
+   = note: see issue #160630 <https://github.com/rust-lang/rust/issues/160630> for more information
+   = help: add `#![feature(f16b)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
+
+error[E0658]: use of unstable library feature `f16b`
+  --> $DIR/feature-gate-f16b.rs:7:13
+   |
+LL |     let _ = f16b::from_bits(0);
+   |             ^^^^^^^^^^^^^^^
+   |
+   = note: see issue #160630 <https://github.com/rust-lang/rust/issues/160630> for more information
+   = help: add `#![feature(f16b)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
+
+error[E0308]: mismatched types
+  --> $DIR/feature-gate-f16b.rs:14:19
+   |
+LL |     let _: f16b = 1.0;
+   |            ----   ^^^ expected `f16b`, found floating-point number
+   |            |
+   |            expected due to this
+
+error[E0658]: use of unstable library feature `f16b`
+  --> $DIR/feature-gate-f16b.rs:18:13
+   |
+LL |     let x = f16b::from_bits(0x3f80);
+   |             ^^^^^^^^^^^^^^^
+   |
+   = note: see issue #160630 <https://github.com/rust-lang/rust/issues/160630> for more information
+   = help: add `#![feature(f16b)]` to the crate attributes to enable
+   = note: this compiler was built on YYYY-MM-DD; consider upgrading it if it is out of date
+
+error[E0369]: cannot add `f16b` to `f16b`
+  --> $DIR/feature-gate-f16b.rs:21:15
+   |
+LL |     let _ = x + x;
+   |             - ^ - f16b
+   |             |
+   |             f16b
+   |
+note: `f16b` does not implement `Add`
+  --> $SRC_DIR/core/src/num/bfloat.rs:LL:COL
+   |
+   = note: `f16b` is defined in another crate
+
+error[E0605]: non-primitive cast: `u16` as `f16b`
+  --> $DIR/feature-gate-f16b.rs:24:13
+   |
+LL |     let _ = 1u16 as f16b;
+   |             ^^^^^^^^^^^^ an `as` expression can only be used to convert between primitive types or to coerce to a specific trait object
+
+error[E0605]: non-primitive cast: `f16b` as `f32`
+  --> $DIR/feature-gate-f16b.rs:28:13
+   |
+LL |     let _ = x as f32;
+   |             ^^^^^^^^ an `as` expression can only be used to convert between primitive types or to coerce to a specific trait object
+   |
+help: consider using the `From` trait instead
+   |
+LL -     let _ = x as f32;
+LL +     let _ = f32::from(x);
+   |
+
+error: aborting due to 12 previous errors
+
+Some errors have detailed explanations: E0308, E0369, E0605, E0658.
+For more information about an error, try `rustc --explain E0308`.
diff --git a/tests/ui/float/f16b-restrictions.rs b/tests/ui/float/f16b-restrictions.rs
new file mode 100644
index 0000000000000..b2dbf1a734be3
--- /dev/null
+++ b/tests/ui/float/f16b-restrictions.rs
@@ -0,0 +1,20 @@
+#![feature(f16b)]
+
+extern crate core;
+
+use core::num::f16b;
+
+fn main() {
+    let _: f16b = 1.0;
+    //~^ ERROR mismatched types
+
+    let x = f16b::from_bits(0x3f80);
+    let _ = x + x;
+    //~^ ERROR cannot add `f16b` to `f16b`
+
+    let _ = 1u16 as f16b;
+    //~^ ERROR non-primitive cast
+
+    let _ = x as f32;
+    //~^ ERROR non-primitive cast
+}
diff --git a/tests/ui/float/f16b-restrictions.stderr b/tests/ui/float/f16b-restrictions.stderr
new file mode 100644
index 0000000000000..540e92d26b384
--- /dev/null
+++ b/tests/ui/float/f16b-restrictions.stderr
@@ -0,0 +1,43 @@
+error[E0308]: mismatched types
+  --> $DIR/f16b-restrictions.rs:8:19
+   |
+LL |     let _: f16b = 1.0;
+   |            ----   ^^^ expected `f16b`, found floating-point number
+   |            |
+   |            expected due to this
+
+error[E0369]: cannot add `f16b` to `f16b`
+  --> $DIR/f16b-restrictions.rs:12:15
+   |
+LL |     let _ = x + x;
+   |             - ^ - f16b
+   |             |
+   |             f16b
+   |
+note: `f16b` does not implement `Add`
+  --> $SRC_DIR/core/src/num/bfloat.rs:LL:COL
+   |
+   = note: `f16b` is defined in another crate
+
+error[E0605]: non-primitive cast: `u16` as `f16b`
+  --> $DIR/f16b-restrictions.rs:15:13
+   |
+LL |     let _ = 1u16 as f16b;
+   |             ^^^^^^^^^^^^ an `as` expression can only be used to convert between primitive types or to coerce to a specific trait object
+
+error[E0605]: non-primitive cast: `f16b` as `f32`
+  --> $DIR/f16b-restrictions.rs:18:13
+   |
+LL |     let _ = x as f32;
+   |             ^^^^^^^^ an `as` expression can only be used to convert between primitive types or to coerce to a specific trait object
+   |
+help: consider using the `From` trait instead
+   |
+LL -     let _ = x as f32;
+LL +     let _ = f32::from(x);
+   |
+
+error: aborting due to 4 previous errors
+
+Some errors have detailed explanations: E0308, E0369, E0605.
+For more information about an error, try `rustc --explain E0308`.
diff --git a/tests/ui/float/f16b.rs b/tests/ui/float/f16b.rs
new file mode 100644
index 0000000000000..7406fdc538dd8
--- /dev/null
+++ b/tests/ui/float/f16b.rs
@@ -0,0 +1,47 @@
+//@ compile-flags: --check-cfg=cfg(target_has_reliable_f16b)
+//@ run-pass
+
+#![feature(f16b, cfg_target_has_reliable_f16b)]
+
+extern crate core;
+
+use core::num::f16b;
+#[cfg(target_has_reliable_f16b)]
+use std::fmt::{Debug, LowerExp, UpperExp};
+
+#[cfg(target_has_reliable_f16b)]
+const ONE: f16b = f16b::from_bits(0x3f80);
+
+#[cfg(target_has_reliable_f16b)]
+const ONE_BITS: u16 = ONE.to_bits();
+
+#[cfg(target_has_reliable_f16b)]
+fn assert_traits<T>()
+where
+    T: Default + Copy + Clone + Debug + LowerExp + UpperExp + PartialEq + PartialOrd,
+{
+}
+
+fn main() {
+    assert_eq!(size_of::<f16b>(), 2);
+    assert_eq!(align_of::<f16b>(), 2);
+
+    #[cfg(target_has_reliable_f16b)]
+    {
+        assert_traits::<f16b>();
+
+        assert_eq!(f16b::default().to_bits(), 0);
+        assert_eq!(ONE_BITS, 0x3f80);
+        assert_eq!(f32::from(ONE).to_bits(), 0x3f80_0000);
+
+        let two = f16b::from_bits(0x4000);
+        let negative_zero = f16b::from_bits(0x8000);
+        let nan = f16b::from_bits(0x7fc0);
+        assert!(ONE < two);
+        assert_eq!(f16b::from_bits(0), negative_zero);
+        assert!(nan != nan);
+        assert_eq!(format!("{ONE:?}"), "1.0");
+        assert_eq!(format!("{ONE:e}"), "1e0");
+        assert_eq!(format!("{ONE:E}"), "1E0");
+    }
+}
diff --git a/tests/ui/parser/f16b.rs b/tests/ui/parser/f16b.rs
new file mode 100644
index 0000000000000..bb6f3737a33d5
--- /dev/null
+++ b/tests/ui/parser/f16b.rs
@@ -0,0 +1,14 @@
+#![feature(f16b)]
+
+extern crate core;
+
+use core::num::f16b;
+
+// `f16b` is a nominal core type, not a floating-point literal type.
+fn main() {
+    let value = f16b::from_bits(0x3f80);
+    let _: f16b = value;
+
+    let _ = 0.0f16b;
+    //~^ ERROR invalid suffix `f16b` for float literal
+}
diff --git a/tests/ui/parser/f16b.stderr b/tests/ui/parser/f16b.stderr
new file mode 100644
index 0000000000000..2d89bbe4d8776
--- /dev/null
+++ b/tests/ui/parser/f16b.stderr
@@ -0,0 +1,10 @@
+error: invalid suffix `f16b` for float literal
+  --> $DIR/f16b.rs:12:13
+   |
+LL |     let _ = 0.0f16b;
+   |             ^^^^^^^ invalid suffix `f16b`
+   |
+   = help: valid suffixes are `f32` and `f64`
+
+error: aborting due to 1 previous error
+

From b0f83b7d0a099fe4af956785ae79aa0edc2d5860 Mon Sep 17 00:00:00 2001
From: Guillaume Gomez <contact@guillaume-gomez.fr>
Date: Fri, 18 Sep 2026 14:55:56 +0200
Subject: [PATCH 19/28] Update `browser-ui-test` version to `0.25.2`

---
 package.json | 2 +-
 yarn.lock    | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/package.json b/package.json
index d9a3ced805ba5..876d81c1d60ef 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "dependencies": {
-    "browser-ui-test": "^0.25.0",
+    "browser-ui-test": "^0.25.2",
     "es-check": "^9.4.4",
     "eslint": "^8.57.1",
     "typescript": "^5.8.3"
diff --git a/yarn.lock b/yarn.lock
index 55c862d1e75dd..e49c9209142f9 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -154,10 +154,10 @@ braces@^3.0.3:
   dependencies:
     fill-range "^7.1.1"
 
-browser-ui-test@^0.25.0:
-  version "0.25.1"
-  resolved "https://registry.yarnpkg.com/browser-ui-test/-/browser-ui-test-0.25.1.tgz#c7f22a5e2b9e51be4ba34df3adf7bd7a9249bce6"
-  integrity sha512-woRwKU1dPBIwYmCI6npox8qlPO0WQ8GZH2YbL39mNkiWymByebiB4EK0PlaGMbmEja0MEqfMQD+d33LCW4S2AA==
+browser-ui-test@^0.25.2:
+  version "0.25.2"
+  resolved "https://registry.yarnpkg.com/browser-ui-test/-/browser-ui-test-0.25.2.tgz#31db7386497b3eef4d79e236bd2091c1162148a9"
+  integrity sha512-74njL1/xjg5UumbhQblWa6oAn/PcaOnlJmoSe9FabJaXrLsI2f8DJ3B8vunvCJwZa90V4Jm94xyg/JPi+l+zZQ==
   dependencies:
     css-unit-converter "^1.1.2"
     pngjs "^3.4.0"

From e7d3a6a9e68afaba5ea299e301c8e57be7626cd6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Le=C3=B3n=20Orell=20Valerian=20Liehr?= <me@fmease.dev>
Date: Thu, 10 Sep 2026 14:11:22 +0200
Subject: [PATCH 20/28] Move parse error recovery from some invalid expr ops
 out of line

---
 compiler/rustc_parse/src/diagnostics.rs       |   4 +-
 compiler/rustc_parse/src/parser/expr.rs       | 388 ++++++++----------
 .../src/parser/expr/diagnostics.rs            |  80 ++++
 3 files changed, 262 insertions(+), 210 deletions(-)
 create mode 100644 compiler/rustc_parse/src/parser/expr/diagnostics.rs

diff --git a/compiler/rustc_parse/src/diagnostics.rs b/compiler/rustc_parse/src/diagnostics.rs
index 829fc5a600e8a..78240b0ee891b 100644
--- a/compiler/rustc_parse/src/diagnostics.rs
+++ b/compiler/rustc_parse/src/diagnostics.rs
@@ -797,7 +797,7 @@ pub(crate) struct EqFieldInit {
 
 #[derive(Diagnostic)]
 #[diag("unexpected token: `...`")]
-pub(crate) struct DotDotDot {
+pub(crate) struct DotDotDotExprOp {
     #[primary_span]
     #[suggestion(
         "use `..` for an exclusive range",
@@ -816,7 +816,7 @@ pub(crate) struct DotDotDot {
 
 #[derive(Diagnostic)]
 #[diag("unexpected token: `<-`")]
-pub(crate) struct LeftArrowOperator {
+pub(crate) struct LArrowExprOp {
     #[primary_span]
     #[suggestion(
         "if you meant to write a comparison against a negative value, add a space in between `<` and `-`",
diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
index 8238a6518e41d..a7bcb93d5b084 100644
--- a/compiler/rustc_parse/src/parser/expr.rs
+++ b/compiler/rustc_parse/src/parser/expr.rs
@@ -34,8 +34,9 @@ use super::{
     AttrWrapper, BlockMode, ClosureSpans, ExpTokenPair, ForceCollect, Parser, PathStyle,
     Restrictions, SemiColonMode, SeqSep, TokenType, Trailing, UsePreAttrPos,
 };
-use crate::diagnostics::ExprParenthesesNeeded;
-use crate::{diagnostics, exp, maybe_recover_from_interpolated_ty_qpath};
+use crate::{exp, maybe_recover_from_interpolated_ty_qpath};
+
+mod diagnostics;
 
 #[derive(Debug)]
 pub(super) enum DestructuredFloat {
@@ -165,74 +166,22 @@ impl<'a> Parser<'a> {
             } {
                 break;
             }
-            // Check for deprecated `...` syntax
-            if self.token == token::DotDotDot && op.node == AssocOp::Range(RangeLimits::Closed) {
-                self.err_dotdotdot_syntax(self.token.span);
-            }
 
-            if self.token == token::LArrow {
-                self.err_larrow_operator(self.token.span);
-            }
+            self.reject_dotdotdot_expr_op();
+            self.reject_larrow_expr_op();
 
             parsed_something = true;
             self.bump();
-            if op.node.is_comparison() {
-                if let Some(expr) = self.check_no_chained_comparison(&lhs, &op)? {
-                    return Ok((expr, parsed_something));
-                }
-            }
 
-            // Look for JS' `===` and `!==` and recover
-            if let AssocOp::Binary(bop @ BinOpKind::Eq | bop @ BinOpKind::Ne) = op.node
-                && self.token == token::Eq
-                && self.prev_token.span.hi() == self.token.span.lo()
+            if op.node.is_comparison()
+                && let Some(expr) = self.check_no_chained_comparison(&lhs, &op)?
             {
-                let sp = op.span.to(self.token.span);
-                let sugg = bop.as_str().into();
-                let invalid = format!("{sugg}=");
-                self.dcx().emit_err(diagnostics::InvalidComparisonOperator {
-                    span: sp,
-                    invalid: invalid.clone(),
-                    sub: diagnostics::InvalidComparisonOperatorSub::Correctable {
-                        span: sp,
-                        invalid,
-                        correct: sugg,
-                    },
-                });
-                self.bump();
+                return Ok((expr, parsed_something));
             }
 
-            // Look for PHP's `<>` and recover
-            if op.node == AssocOp::Binary(BinOpKind::Lt)
-                && self.token == token::Gt
-                && self.prev_token.span.hi() == self.token.span.lo()
-            {
-                let sp = op.span.to(self.token.span);
-                self.dcx().emit_err(diagnostics::InvalidComparisonOperator {
-                    span: sp,
-                    invalid: "<>".into(),
-                    sub: diagnostics::InvalidComparisonOperatorSub::Correctable {
-                        span: sp,
-                        invalid: "<>".into(),
-                        correct: "!=".into(),
-                    },
-                });
-                self.bump();
-            }
-
-            // Look for C++'s `<=>` and recover
-            if op.node == AssocOp::Binary(BinOpKind::Le)
-                && self.token == token::Gt
-                && self.prev_token.span.hi() == self.token.span.lo()
-            {
-                let sp = op.span.to(self.token.span);
-                self.dcx().emit_err(diagnostics::InvalidComparisonOperator {
-                    span: sp,
-                    invalid: "<=>".into(),
-                    sub: diagnostics::InvalidComparisonOperatorSub::Spaceship(sp),
-                });
-                self.bump();
-            }
+            self.recover_from_strict_eq_op(op);
+            self.recover_from_diamond_ne_op(op);
+            self.recover_from_spaceship_cmp_op(op);
 
             if self.prev_token == token::Plus
                 && self.token == token::Plus
@@ -337,10 +286,10 @@ impl<'a> Parser<'a> {
     /// but the next token implies this should be parsed as an expression.
     /// For example: `if let Some(x) = x { x } else { 0 } / 2`.
     fn error_found_expr_would_be_stmt(&self, lhs: &Expr) {
-        self.dcx().emit_err(diagnostics::FoundExprWouldBeStmt {
+        self.dcx().emit_err(crate::diagnostics::FoundExprWouldBeStmt {
             span: self.token.span,
             token: pprust::token_to_string(&self.token),
-            suggestion: ExprParenthesesNeeded::surrounding(lhs.span),
+            suggestion: crate::diagnostics::ExprParenthesesNeeded::surrounding(lhs.span),
         });
     }
 
@@ -377,18 +326,22 @@ impl<'a> Parser<'a> {
             (None, Some((Ident { name: sym::and, span }, IdentIsRaw::No)))
                 if self.may_recover() =>
             {
-                self.dcx().emit_err(diagnostics::InvalidLogicalOperator {
+                self.dcx().emit_err(crate::diagnostics::InvalidLogicalOperator {
                     span: self.token.span,
                     incorrect: "and".into(),
-                    sub: diagnostics::InvalidLogicalOperatorSub::Conjunction(self.token.span),
+                    sub: crate::diagnostics::InvalidLogicalOperatorSub::Conjunction(
+                        self.token.span,
+                    ),
                 });
                 (AssocOp::Binary(BinOpKind::And), span)
             }
             (None, Some((Ident { name: sym::or, span }, IdentIsRaw::No))) if self.may_recover() => {
-                self.dcx().emit_err(diagnostics::InvalidLogicalOperator {
+                self.dcx().emit_err(crate::diagnostics::InvalidLogicalOperator {
                     span: self.token.span,
                     incorrect: "or".into(),
-                    sub: diagnostics::InvalidLogicalOperatorSub::Disjunction(self.token.span),
+                    sub: crate::diagnostics::InvalidLogicalOperatorSub::Disjunction(
+                        self.token.span,
+                    ),
                 });
                 (AssocOp::Binary(BinOpKind::Or), span)
             }
@@ -441,14 +394,11 @@ impl<'a> Parser<'a> {
     /// Parses prefix-forms of range notation: `..expr`, `..`, `..=expr`.
     fn parse_expr_prefix_range(&mut self, attrs: AttrWrapper) -> PResult<'a, Box<Expr>> {
         if !attrs.is_empty() {
-            let err = diagnostics::DotDotRangeAttribute { span: self.token.span };
+            let err = crate::diagnostics::DotDotRangeAttribute { span: self.token.span };
             self.dcx().emit_err(err);
         }
 
-        // Check for deprecated `...` syntax.
-        if self.token == token::DotDotDot {
-            self.err_dotdotdot_syntax(self.token.span);
-        }
+        self.reject_dotdotdot_expr_op();
 
         debug_assert!(
             self.token.is_range_separator(),
@@ -513,7 +463,7 @@ impl<'a> Parser<'a> {
             }
             // `+lit`
             token::Plus if this.look_ahead(1, |tok| tok.is_numeric_lit()) => {
-                let mut err = diagnostics::LeadingPlusNotSupported {
+                let mut err = crate::diagnostics::LeadingPlusNotSupported {
                     span: lo,
                     remove_plus: None,
                     add_parentheses: None,
@@ -521,7 +471,8 @@ impl<'a> Parser<'a> {
 
                 // a block on the LHS might have been intended to be an expression instead
                 if let Some(sp) = this.psess.ambiguous_block_expr_parse.borrow().get(&lo) {
-                    err.add_parentheses = Some(ExprParenthesesNeeded::surrounding(*sp));
+                    err.add_parentheses =
+                        Some(crate::diagnostics::ExprParenthesesNeeded::surrounding(*sp));
                 } else {
                     err.remove_plus = Some(lo);
                 }
@@ -574,7 +525,7 @@ impl<'a> Parser<'a> {
 
     /// Recover on `~expr` in favor of `!expr`.
     fn recover_tilde_expr(&mut self, lo: Span) -> PResult<'a, (Span, ExprKind)> {
-        self.dcx().emit_err(diagnostics::TildeAsUnaryOperator(lo));
+        self.dcx().emit_err(crate::diagnostics::TildeAsUnaryOperator(lo));
 
         self.parse_expr_unary(lo, UnOp::Not)
     }
@@ -605,14 +556,14 @@ impl<'a> Parser<'a> {
         let negated_token = self.look_ahead(1, |t| *t);
 
         let sub_diag = if negated_token.is_numeric_lit() {
-            diagnostics::NotAsNegationOperatorSub::SuggestNotBitwise
+            crate::diagnostics::NotAsNegationOperatorSub::SuggestNotBitwise
         } else if negated_token.is_bool_lit() {
-            diagnostics::NotAsNegationOperatorSub::SuggestNotLogical
+            crate::diagnostics::NotAsNegationOperatorSub::SuggestNotLogical
         } else {
-            diagnostics::NotAsNegationOperatorSub::SuggestNotDefault
+            crate::diagnostics::NotAsNegationOperatorSub::SuggestNotDefault
         };
 
-        self.dcx().emit_err(diagnostics::NotAsNegationOperator {
+        self.dcx().emit_err(crate::diagnostics::NotAsNegationOperator {
             negated: negated_token.span,
             negated_desc: super::token_descr(&negated_token),
             // Span the `not` plus trailing whitespace to avoid
@@ -683,7 +634,7 @@ impl<'a> Parser<'a> {
                         match self.parse_expr_labeled(label, false) {
                             Ok(expr) => {
                                 type_err.cancel();
-                                self.dcx().emit_err(diagnostics::MalformedLoopLabel {
+                                self.dcx().emit_err(crate::diagnostics::MalformedLoopLabel {
                                     span: label.ident.span,
                                     suggestion: label.ident.span.shrink_to_lo(),
                                 });
@@ -709,23 +660,24 @@ impl<'a> Parser<'a> {
 
                         let args_span = self.look_ahead(1, |t| t.span).to(span_after_type);
                         match self.token.kind {
-                            token::Lt => {
-                                self.dcx().emit_err(diagnostics::ComparisonInterpretedAsGeneric {
+                            token::Lt => self.dcx().emit_err(
+                                crate::diagnostics::ComparisonInterpretedAsGeneric {
                                     comparison: self.token.span,
                                     r#type: pprust::path_to_string(&path),
                                     args: args_span,
-                                    suggestion: diagnostics::ComparisonInterpretedAsGenericSugg {
-                                        left: expr.span.shrink_to_lo(),
-                                        right: expr.span.shrink_to_hi(),
-                                    },
-                                })
-                            }
+                                    suggestion:
+                                        crate::diagnostics::ComparisonInterpretedAsGenericSugg {
+                                            left: expr.span.shrink_to_lo(),
+                                            right: expr.span.shrink_to_hi(),
+                                        },
+                                },
+                            ),
                             token::Shl => {
-                                self.dcx().emit_err(diagnostics::ShiftInterpretedAsGeneric {
+                                self.dcx().emit_err(crate::diagnostics::ShiftInterpretedAsGeneric {
                                     shift: self.token.span,
                                     r#type: pprust::path_to_string(&path),
                                     args: args_span,
-                                    suggestion: diagnostics::ShiftInterpretedAsGenericSugg {
+                                    suggestion: crate::diagnostics::ShiftInterpretedAsGenericSugg {
                                         left: expr.span.shrink_to_lo(),
                                         right: expr.span.shrink_to_hi(),
                                     },
@@ -835,8 +787,10 @@ impl<'a> Parser<'a> {
     }
 
     fn error_remove_borrow_lifetime(&self, span: Span, lt_span: Span) {
-        self.dcx()
-            .emit_err(diagnostics::LifetimeInBorrowExpression { span, lifetime_span: lt_span });
+        self.dcx().emit_err(crate::diagnostics::LifetimeInBorrowExpression {
+            span,
+            lifetime_span: lt_span,
+        });
     }
 
     /// Parse `mut?` or `[ raw | pin ] [ const | mut ]`.
@@ -895,7 +849,7 @@ impl<'a> Parser<'a> {
                 // Recovery for `expr->suffix`.
                 self.bump();
                 let span = self.prev_token.span;
-                self.dcx().emit_err(diagnostics::ExprRArrowCall { span });
+                self.dcx().emit_err(crate::diagnostics::ExprRArrowCall { span });
                 true
             } else {
                 self.eat(exp!(Dot))
@@ -1018,7 +972,7 @@ impl<'a> Parser<'a> {
             }
             _ => (span, actual),
         };
-        self.dcx().emit_err(diagnostics::UnexpectedTokenAfterDot { span, actual });
+        self.dcx().emit_err(crate::diagnostics::UnexpectedTokenAfterDot { span, actual });
     }
 
     /// We need an identifier or integer, but the next token is a float.
@@ -1135,7 +1089,7 @@ impl<'a> Parser<'a> {
                     // Parse this both to give helpful error messages and to
                     // verify it can be done with this parser setup.
                     ExprKind::Index(ref left, ref _right, span) => {
-                        self.dcx().emit_err(diagnostics::ArrayIndexInOffsetOf(span));
+                        self.dcx().emit_err(crate::diagnostics::ArrayIndexInOffsetOf(span));
                         current = left;
                     }
                     ExprKind::Lit(token::Lit {
@@ -1144,10 +1098,12 @@ impl<'a> Parser<'a> {
                         suffix,
                     }) => {
                         if let Some(suffix) = suffix {
-                            self.dcx().emit_err(diagnostics::InvalidLiteralSuffixOnTupleIndex {
-                                span: current.span,
-                                suffix,
-                            });
+                            self.dcx().emit_err(
+                                crate::diagnostics::InvalidLiteralSuffixOnTupleIndex {
+                                    span: current.span,
+                                    suffix,
+                                },
+                            );
                         }
                         match self.break_up_float(symbol, current.span) {
                             // 1e2
@@ -1187,14 +1143,15 @@ impl<'a> Parser<'a> {
                                 fields.insert(start_idx, *ident)
                             }
                             _ => {
-                                self.dcx().emit_err(diagnostics::InvalidOffsetOf(current.span));
+                                self.dcx()
+                                    .emit_err(crate::diagnostics::InvalidOffsetOf(current.span));
                                 break;
                             }
                         }
                         break;
                     }
                     _ => {
-                        self.dcx().emit_err(diagnostics::InvalidOffsetOf(current.span));
+                        self.dcx().emit_err(crate::diagnostics::InvalidOffsetOf(current.span));
                         break;
                     }
                 }
@@ -1204,12 +1161,12 @@ impl<'a> Parser<'a> {
                 break;
             } else if trailing_dot.is_none() {
                 // This loop should only repeat if there is a trailing dot.
-                self.dcx().emit_err(diagnostics::InvalidOffsetOf(self.token.span));
+                self.dcx().emit_err(crate::diagnostics::InvalidOffsetOf(self.token.span));
                 break;
             }
         }
         if let Some(dot) = trailing_dot {
-            self.dcx().emit_err(diagnostics::InvalidOffsetOf(dot));
+            self.dcx().emit_err(crate::diagnostics::InvalidOffsetOf(dot));
         }
         Ok(fields.into_iter().collect())
     }
@@ -1223,7 +1180,7 @@ impl<'a> Parser<'a> {
         suffix: Option<Symbol>,
     ) -> Box<Expr> {
         if let Some(suffix) = suffix {
-            self.dcx().emit_err(diagnostics::InvalidLiteralSuffixOnTupleIndex {
+            self.dcx().emit_err(crate::diagnostics::InvalidLiteralSuffixOnTupleIndex {
                 span: ident_span,
                 suffix,
             });
@@ -1310,14 +1267,14 @@ impl<'a> Parser<'a> {
                             err.cancel();
                             let type_str = pprust::path_to_string(&path);
                             self.dcx()
-                                .create_err(diagnostics::ParenthesesWithStructFields {
+                                .create_err(crate::diagnostics::ParenthesesWithStructFields {
                                     span,
-                                    braces_for_struct: diagnostics::BracesForStructLiteral {
+                                    braces_for_struct: crate::diagnostics::BracesForStructLiteral {
                                         first: open_paren,
                                         second: close_paren,
                                         r#type: type_str.clone(),
                                     },
-                                    no_fields_for_fn: diagnostics::NoFieldsForFnCall {
+                                    no_fields_for_fn: crate::diagnostics::NoFieldsForFnCall {
                                         r#type: type_str,
                                         fields: fields
                                             .into_iter()
@@ -1419,7 +1376,7 @@ impl<'a> Parser<'a> {
             if let Some(args) = seg.args {
                 // See `StashKey::GenericInFieldExpr` for more info on why we stash this.
                 self.dcx()
-                    .create_err(diagnostics::FieldExpressionWithGeneric(args.span()))
+                    .create_err(crate::diagnostics::FieldExpressionWithGeneric(args.span()))
                     .stash(seg.ident.span, StashKey::GenericInFieldExpr);
             }
 
@@ -1491,7 +1448,9 @@ impl<'a> Parser<'a> {
                     // If the input is something like `if a { 1 } else { 2 } | if a { 3 } else { 4 }`
                     // then suggest parens around the lhs.
                     if let Some(sp) = this.psess.ambiguous_block_expr_parse.borrow().get(&lo) {
-                        err.subdiagnostic(ExprParenthesesNeeded::surrounding(*sp));
+                        err.subdiagnostic(crate::diagnostics::ExprParenthesesNeeded::surrounding(
+                            *sp,
+                        ));
                     }
                     err
                 })
@@ -1689,7 +1648,8 @@ impl<'a> Parser<'a> {
         let (span, kind) = if self.eat(exp!(Bang)) {
             // MACRO INVOCATION expression
             if qself.is_some() {
-                self.dcx().emit_err(diagnostics::MacroInvocationWithQualifiedPath(path.span));
+                self.dcx()
+                    .emit_err(crate::diagnostics::MacroInvocationWithQualifiedPath(path.span));
             }
             let lo = path.span;
             let mac = Box::new(MacCall { path, args: self.parse_delim_args()? });
@@ -1734,7 +1694,7 @@ impl<'a> Parser<'a> {
         {
             let (lit, _) =
                 self.recover_unclosed_char(label_.ident, Parser::mk_token_lit_char, |self_| {
-                    self_.dcx().create_err(diagnostics::UnexpectedTokenAfterLabel {
+                    self_.dcx().create_err(crate::diagnostics::UnexpectedTokenAfterLabel {
                         span: self_.token.span,
                         remove_label: None,
                         enclose_in_block: None,
@@ -1746,7 +1706,7 @@ impl<'a> Parser<'a> {
             && (self.check_noexpect(&TokenKind::Comma) || self.check_noexpect(&TokenKind::Gt))
         {
             // We're probably inside of a `Path<'a>` that needs a turbofish
-            let guar = self.dcx().emit_err(diagnostics::UnexpectedTokenAfterLabel {
+            let guar = self.dcx().emit_err(crate::diagnostics::UnexpectedTokenAfterLabel {
                 span: self.token.span,
                 remove_label: None,
                 enclose_in_block: None,
@@ -1754,7 +1714,7 @@ impl<'a> Parser<'a> {
             consume_colon = false;
             Ok(self.mk_expr_err(lo, guar))
         } else {
-            let mut err = diagnostics::UnexpectedTokenAfterLabel {
+            let mut err = crate::diagnostics::UnexpectedTokenAfterLabel {
                 span: self.token.span,
                 remove_label: None,
                 enclose_in_block: None,
@@ -1791,7 +1751,7 @@ impl<'a> Parser<'a> {
                     return expr;
                 }
 
-                err.enclose_in_block = Some(diagnostics::UnexpectedTokenAfterLabelSugg {
+                err.enclose_in_block = Some(crate::diagnostics::UnexpectedTokenAfterLabelSugg {
                     left: span.shrink_to_lo(),
                     right: span.shrink_to_hi(),
                 });
@@ -1807,7 +1767,7 @@ impl<'a> Parser<'a> {
         }?;
 
         if !ate_colon && consume_colon {
-            self.dcx().emit_err(diagnostics::RequireColonAfterLabeledExpression {
+            self.dcx().emit_err(crate::diagnostics::RequireColonAfterLabeledExpression {
                 span: expr.span,
                 label: lo,
                 label_end: lo.between(tok_sp),
@@ -1856,7 +1816,7 @@ impl<'a> Parser<'a> {
         self.bump(); // `catch`
 
         let span = lo.to(self.prev_token.span);
-        self.dcx().emit_err(diagnostics::DoCatchSyntaxRemoved { span });
+        self.dcx().emit_err(crate::diagnostics::DoCatchSyntaxRemoved { span });
 
         self.parse_try_block(lo)
     }
@@ -1916,9 +1876,9 @@ impl<'a> Parser<'a> {
             // The value expression can be a labeled loop, see issue #86948, e.g.:
             // `loop { break 'label: loop { break 'label 42; }; }`
             let lexpr = self.parse_expr_labeled(label, true)?;
-            self.dcx().emit_err(diagnostics::LabeledLoopInBreak {
+            self.dcx().emit_err(crate::diagnostics::LabeledLoopInBreak {
                 span: lexpr.span,
-                sub: diagnostics::WrapInParentheses::Expression {
+                sub: crate::diagnostics::WrapInParentheses::Expression {
                     left: lexpr.span.shrink_to_lo(),
                     right: lexpr.span.shrink_to_hi(),
                 },
@@ -1945,8 +1905,8 @@ impl<'a> Parser<'a> {
                         BREAK_WITH_LABEL_AND_LOOP,
                         lo.to(expr.span),
                         ast::CRATE_NODE_ID,
-                        diagnostics::BreakWithLabelAndLoop {
-                            sub: diagnostics::BreakWithLabelAndLoopSub {
+                        crate::diagnostics::BreakWithLabelAndLoop {
+                            sub: crate::diagnostics::BreakWithLabelAndLoopSub {
                                 left: span.shrink_to_lo(),
                                 right: span.shrink_to_hi(),
                             },
@@ -2028,8 +1988,9 @@ impl<'a> Parser<'a> {
         self.bump(); // `#`
 
         let Some((ident, IdentIsRaw::No)) = self.token.ident() else {
-            let err =
-                self.dcx().create_err(diagnostics::ExpectedBuiltinIdent { span: self.token.span });
+            let err = self
+                .dcx()
+                .create_err(crate::diagnostics::ExpectedBuiltinIdent { span: self.token.span });
             return Err(err);
         };
         self.psess.gated_spans.gate(sym::builtin_syntax, ident.span);
@@ -2039,7 +2000,7 @@ impl<'a> Parser<'a> {
         let ret = if let Some(res) = parse(self, lo, ident)? {
             Ok(res)
         } else {
-            let err = self.dcx().create_err(diagnostics::UnknownBuiltinConstruct {
+            let err = self.dcx().create_err(crate::diagnostics::UnknownBuiltinConstruct {
                 span: lo.to(ident.span),
                 name: ident,
             });
@@ -2188,7 +2149,7 @@ impl<'a> Parser<'a> {
                 }
             });
             if let Some(recovered) = recovered {
-                self.dcx().emit_err(diagnostics::FloatLiteralRequiresIntegerPart {
+                self.dcx().emit_err(crate::diagnostics::FloatLiteralRequiresIntegerPart {
                     span: recovered.span,
                     suggestion: recovered.span.shrink_to_lo(),
                 });
@@ -2322,9 +2283,9 @@ impl<'a> Parser<'a> {
         let mut snapshot = self.create_snapshot_for_diagnostic();
         match snapshot.parse_expr_array_or_repeat(exp!(CloseBrace)) {
             Ok(arr) => {
-                let guar = self.dcx().emit_err(diagnostics::ArrayBracketsInsteadOfBraces {
+                let guar = self.dcx().emit_err(crate::diagnostics::ArrayBracketsInsteadOfBraces {
                     span: arr.span,
-                    sub: diagnostics::ArrayBracketsInsteadOfBracesSugg {
+                    sub: crate::diagnostics::ArrayBracketsInsteadOfBracesSugg {
                         left: lo,
                         right: snapshot.prev_token.span,
                     },
@@ -2370,7 +2331,7 @@ impl<'a> Parser<'a> {
                         .span_to_snippet(snapshot.token.span)
                         .is_ok_and(|snippet| snippet == "]") =>
                 {
-                    return Err(self.dcx().create_err(diagnostics::MissingSemicolonBeforeArray {
+                    return Err(self.dcx().create_err(crate::diagnostics::MissingSemicolonBeforeArray {
                         open_delim: open_delim_span,
                         semicolon: prev_span.shrink_to_hi(),
                     }));
@@ -2396,10 +2357,10 @@ impl<'a> Parser<'a> {
         }
 
         if self.token.is_metavar_block() {
-            self.dcx().emit_err(diagnostics::InvalidBlockMacroSegment {
+            self.dcx().emit_err(crate::diagnostics::InvalidBlockMacroSegment {
                 span: self.token.span,
                 context: lo.to(self.token.span),
-                wrap: diagnostics::WrapInExplicitBlock {
+                wrap: crate::diagnostics::WrapInExplicitBlock {
                     lo: self.token.span.shrink_to_lo(),
                     hi: self.token.span.shrink_to_hi(),
                 },
@@ -2571,9 +2532,9 @@ impl<'a> Parser<'a> {
             // Check for `move async` and recover
             if self.check_keyword(exp!(Async)) {
                 let move_async_span = self.token.span.with_lo(self.prev_token.span.data().lo);
-                Err(self
-                    .dcx()
-                    .create_err(diagnostics::AsyncMoveOrderIncorrect { span: move_async_span }))
+                Err(self.dcx().create_err(crate::diagnostics::AsyncMoveOrderIncorrect {
+                    span: move_async_span,
+                }))
             } else {
                 Ok(CaptureBy::Value { move_kw: move_kw_span })
             }
@@ -2583,9 +2544,9 @@ impl<'a> Parser<'a> {
             // Check for `use async` and recover
             if self.check_keyword(exp!(Async)) {
                 let use_async_span = self.token.span.with_lo(self.prev_token.span.data().lo);
-                Err(self
-                    .dcx()
-                    .create_err(diagnostics::AsyncUseOrderIncorrect { span: use_async_span }))
+                Err(self.dcx().create_err(crate::diagnostics::AsyncUseOrderIncorrect {
+                    span: use_async_span,
+                }))
             } else {
                 Ok(CaptureBy::Use { use_kw: use_kw_span })
             }
@@ -2667,10 +2628,10 @@ impl<'a> Parser<'a> {
                 ExprKind::Binary(Spanned { span: binop_span, .. }, _, right)
                     if let ExprKind::Block(_, None) = right.kind =>
                 {
-                    let guar = this.dcx().emit_err(diagnostics::IfExpressionMissingThenBlock {
+                    let guar = this.dcx().emit_err(crate::diagnostics::IfExpressionMissingThenBlock {
                         if_span: lo,
                         missing_then_block_sub:
-                            diagnostics::IfExpressionMissingThenBlockSub::UnfinishedCondition(
+                            crate::diagnostics::IfExpressionMissingThenBlockSub::UnfinishedCondition(
                                 cond_span.shrink_to_lo().to(*binop_span),
                             ),
                         let_else_sub: None,
@@ -2678,10 +2639,11 @@ impl<'a> Parser<'a> {
                     std::mem::replace(right, this.mk_expr_err(binop_span.shrink_to_hi(), guar))
                 }
                 ExprKind::Block(_, None) => {
-                    let guar = this.dcx().emit_err(diagnostics::IfExpressionMissingCondition {
-                        if_span: lo.with_neighbor(cond.span).shrink_to_hi(),
-                        block_span: self.psess.source_map().start_point(cond_span),
-                    });
+                    let guar =
+                        this.dcx().emit_err(crate::diagnostics::IfExpressionMissingCondition {
+                            if_span: lo.with_neighbor(cond.span).shrink_to_hi(),
+                            block_span: self.psess.source_map().start_point(cond_span),
+                        });
                     std::mem::replace(&mut cond, this.mk_expr_err(cond_span.shrink_to_hi(), guar))
                 }
                 _ => {
@@ -2699,13 +2661,14 @@ impl<'a> Parser<'a> {
             if let Some(block) = recover_block_from_condition(self) {
                 block
             } else {
-                let let_else_sub = matches!(cond.kind, ExprKind::Let(..))
-                    .then(|| diagnostics::IfExpressionLetSomeSub { if_span: lo.until(cond_span) });
+                let let_else_sub = matches!(cond.kind, ExprKind::Let(..)).then(|| {
+                    crate::diagnostics::IfExpressionLetSomeSub { if_span: lo.until(cond_span) }
+                });
 
-                let guar = self.dcx().emit_err(diagnostics::IfExpressionMissingThenBlock {
+                let guar = self.dcx().emit_err(crate::diagnostics::IfExpressionMissingThenBlock {
                     if_span: lo,
                     missing_then_block_sub:
-                        diagnostics::IfExpressionMissingThenBlockSub::AddThenBlock(
+                        crate::diagnostics::IfExpressionMissingThenBlockSub::AddThenBlock(
                             cond_span.shrink_to_hi(),
                         ),
                     let_else_sub,
@@ -2798,9 +2761,9 @@ impl<'a> Parser<'a> {
     /// Parses a `let $pat = $expr` pseudo-expression.
     fn parse_expr_let(&mut self, restrictions: Restrictions) -> PResult<'a, Box<Expr>> {
         let recovered: Recovered = if !restrictions.contains(Restrictions::ALLOW_LET) {
-            let err = diagnostics::ExpectedExpressionFoundLet {
+            let err = crate::diagnostics::ExpectedExpressionFoundLet {
                 span: self.token.span,
-                reason: diagnostics::ForbiddenLetReason::OtherForbidden,
+                reason: crate::diagnostics::ForbiddenLetReason::OtherForbidden,
                 missing_let: None,
                 comparison: None,
             };
@@ -2822,7 +2785,7 @@ impl<'a> Parser<'a> {
             CommaRecoveryMode::LikelyTuple,
         )?;
         if self.token == token::EqEq {
-            self.dcx().emit_err(diagnostics::ExpectedEqForLetExpr {
+            self.dcx().emit_err(crate::diagnostics::ExpectedEqForLetExpr {
                 span: self.token.span,
                 sugg_span: self.token.span,
             });
@@ -2888,7 +2851,7 @@ impl<'a> Parser<'a> {
                             || matches!(cond.kind, ExprKind::MacCall(..)))
                     =>
                 {
-                    self.dcx().emit_err(diagnostics::ExpectedElseBlock {
+                    self.dcx().emit_err(crate::diagnostics::ExpectedElseBlock {
                         first_tok_span,
                         first_tok,
                         else_span,
@@ -2924,7 +2887,7 @@ impl<'a> Parser<'a> {
             let attributes = x0.span.until(branch_span);
             let last = xn.span;
             let ctx = if is_ctx_else { "else" } else { "if" };
-            self.dcx().emit_err(diagnostics::OuterAttributeNotAllowedOnIfElse {
+            self.dcx().emit_err(crate::diagnostics::OuterAttributeNotAllowedOnIfElse {
                 last,
                 branch_span,
                 ctx_span,
@@ -2939,7 +2902,7 @@ impl<'a> Parser<'a> {
             && let BinOpKind::And = binop
             && let ExprKind::If(cond, ..) = &right.kind
         {
-            Err(self.dcx().create_err(diagnostics::UnexpectedIfWithIf(
+            Err(self.dcx().create_err(crate::diagnostics::UnexpectedIfWithIf(
                 binop_span.shrink_to_hi().to(cond.span.shrink_to_lo()),
             )))
         } else {
@@ -2989,12 +2952,12 @@ impl<'a> Parser<'a> {
                     let right = self.prev_token.span.between(self.look_ahead(1, |t| t.span));
                     self.bump(); // )
                     err.cancel();
-                    self.dcx().emit_err(diagnostics::ParenthesesInForHead {
+                    self.dcx().emit_err(crate::diagnostics::ParenthesesInForHead {
                         span,
                         // With e.g. `for (x) in y)` this would replace `(x) in y)`
                         // with `x) in y)` which is syntactically invalid.
                         // However, this is prevented before we get here.
-                        sugg: diagnostics::ParenthesesInForHeadSugg { left, right },
+                        sugg: crate::diagnostics::ParenthesesInForHeadSugg { left, right },
                     });
                     Ok((self.mk_pat(start_span.to(right), ast::PatKind::Wild), expr))
                 } else {
@@ -3029,7 +2992,7 @@ impl<'a> Parser<'a> {
             && self.token.kind != token::OpenBrace
             && self.may_recover()
         {
-            let guar = self.dcx().emit_err(diagnostics::MissingExpressionInForLoop {
+            let guar = self.dcx().emit_err(crate::diagnostics::MissingExpressionInForLoop {
                 span: expr.span.shrink_to_lo(),
             });
             let err_expr = self.mk_expr(expr.span, ExprKind::Err(guar));
@@ -3071,7 +3034,7 @@ impl<'a> Parser<'a> {
             let else_span = self.token.span;
             self.bump();
             let else_clause = self.parse_expr_else()?;
-            self.dcx().emit_err(diagnostics::LoopElseNotSupported {
+            self.dcx().emit_err(crate::diagnostics::LoopElseNotSupported {
                 span: else_span.to(else_clause.span),
                 loop_kind,
                 loop_kw,
@@ -3085,18 +3048,18 @@ impl<'a> Parser<'a> {
             // Possibly using JS syntax (#75311).
             let span = self.token.span;
             self.bump();
-            (span, Some(diagnostics::MissingInInForLoopSub::InNotOf(span)))
+            (span, Some(crate::diagnostics::MissingInInForLoopSub::InNotOf(span)))
         } else if self.eat(exp!(Eq)) {
             let span = self.prev_token.span;
-            (span, Some(diagnostics::MissingInInForLoopSub::InNotEq(span)))
+            (span, Some(crate::diagnostics::MissingInInForLoopSub::InNotEq(span)))
         } else {
             let span = self.prev_token.span.between(self.token.span);
             let sub = (!self.for_loop_head_has_in())
-                .then_some(diagnostics::MissingInInForLoopSub::AddIn(span));
+                .then_some(crate::diagnostics::MissingInInForLoopSub::AddIn(span));
             (span, sub)
         };
 
-        self.dcx().emit_err(diagnostics::MissingInInForLoop { span, sub });
+        self.dcx().emit_err(crate::diagnostics::MissingInInForLoop { span, sub });
     }
 
     /// Whether the `for` loop header already contains an `in` before its body.
@@ -3166,7 +3129,7 @@ impl<'a> Parser<'a> {
         if let Some((ident, is_raw)) = self.token.lifetime() {
             // Disallow `'fn`, but with a better error message than `expect_lifetime`.
             if is_raw == IdentIsRaw::No && ident.without_first_quote().is_reserved() {
-                self.dcx().emit_err(diagnostics::KeywordLabel { span: ident.span });
+                self.dcx().emit_err(crate::diagnostics::KeywordLabel { span: ident.span });
             }
 
             self.bump();
@@ -3263,18 +3226,20 @@ impl<'a> Parser<'a> {
         let err = |this: &Parser<'_>, stmts: Vec<ast::Stmt>| {
             let span = stmts[0].span.to(stmts[stmts.len() - 1].span);
 
-            let guar = this.dcx().emit_err(diagnostics::MatchArmBodyWithoutBraces {
+            let guar = this.dcx().emit_err(crate::diagnostics::MatchArmBodyWithoutBraces {
                 statements: span,
                 arrow: arrow_span,
                 num_statements: stmts.len(),
                 sub: if stmts.len() > 1 {
-                    diagnostics::MatchArmBodyWithoutBracesSugg::AddBraces {
+                    crate::diagnostics::MatchArmBodyWithoutBracesSugg::AddBraces {
                         left: span.shrink_to_lo(),
                         right: span.shrink_to_hi(),
                         num_statements: stmts.len(),
                     }
                 } else {
-                    diagnostics::MatchArmBodyWithoutBracesSugg::UseComma { semicolon: semi_sp }
+                    crate::diagnostics::MatchArmBodyWithoutBracesSugg::UseComma {
+                        semicolon: semi_sp,
+                    }
                 },
             });
             (span, guar)
@@ -3492,7 +3457,7 @@ impl<'a> Parser<'a> {
                         .is_ok();
                     if pattern_follows && snapshot.check(exp!(FatArrow)) {
                         err.cancel();
-                        let guar = this.dcx().emit_err(diagnostics::MissingCommaAfterMatchArm {
+                        let guar = this.dcx().emit_err(crate::diagnostics::MissingCommaAfterMatchArm {
                             span: arm_span.shrink_to_hi(),
                         });
                         return Ok(Recovered::Yes(guar));
@@ -3585,9 +3550,9 @@ impl<'a> Parser<'a> {
                 checker.visit_expr(&mut guard.cond);
 
                 let right = self.prev_token.span;
-                self.dcx().emit_err(diagnostics::ParenthesesInMatchPat {
+                self.dcx().emit_err(crate::diagnostics::ParenthesesInMatchPat {
                     span: vec![left, right],
-                    sugg: diagnostics::ParenthesesInMatchPatSugg { left, right },
+                    sugg: crate::diagnostics::ParenthesesInMatchPatSugg { left, right },
                 });
 
                 if let Some(guar) = checker.found_incorrect_let_chain {
@@ -3664,7 +3629,9 @@ impl<'a> Parser<'a> {
 
         let (attrs, body) = self.parse_inner_attrs_and_block(None)?;
         if self.eat_keyword(exp!(Catch)) {
-            Err(self.dcx().create_err(diagnostics::CatchAfterTry { span: self.prev_token.span }))
+            Err(self
+                .dcx()
+                .create_err(crate::diagnostics::CatchAfterTry { span: self.prev_token.span }))
         } else {
             let span = span_lo.to(body.span);
             let gate_sym =
@@ -3767,9 +3734,9 @@ impl<'a> Parser<'a> {
                 match self.parse_expr_struct(qself.clone(), path.clone(), false) {
                     Ok(expr) => {
                         // This is a struct literal, but we don't accept them here.
-                        self.dcx().emit_err(diagnostics::StructLiteralNotAllowedHere {
+                        self.dcx().emit_err(crate::diagnostics::StructLiteralNotAllowedHere {
                             span: expr.span,
-                            sub: diagnostics::StructLiteralNotAllowedHereSugg {
+                            sub: crate::diagnostics::StructLiteralNotAllowedHereSugg {
                                 left: path.span.shrink_to_lo(),
                                 right: expr.span.shrink_to_hi(),
                             },
@@ -3811,10 +3778,12 @@ impl<'a> Parser<'a> {
             )?;
 
             let guar = if is_underscore_entry_point {
-                self.dcx().create_err(diagnostics::StructLiteralPlaceholderPath { span }).emit()
+                self.dcx()
+                    .create_err(crate::diagnostics::StructLiteralPlaceholderPath { span })
+                    .emit()
             } else {
                 self.dcx()
-                    .create_err(diagnostics::StructLiteralWithoutPathLate {
+                    .create_err(crate::diagnostics::StructLiteralWithoutPathLate {
                         span: expr.span,
                         suggestion_span: expr.span.shrink_to_lo(),
                     })
@@ -3846,8 +3815,8 @@ impl<'a> Parser<'a> {
         let in_if_guard = self.restrictions.contains(Restrictions::IN_IF_GUARD);
 
         let async_block_err = |e: &mut Diag<'_>, span: Span| {
-            diagnostics::AsyncBlockIn2015 { span }.add_to_diag(e);
-            diagnostics::HelpUseLatestEdition::new().add_to_diag(e);
+            crate::diagnostics::AsyncBlockIn2015 { span }.add_to_diag(e);
+            crate::diagnostics::HelpUseLatestEdition::new().add_to_diag(e);
         };
 
         while self.token != close.tok {
@@ -4029,7 +3998,7 @@ impl<'a> Parser<'a> {
         if self.token != token::Comma {
             return;
         }
-        self.dcx().emit_err(diagnostics::CommaAfterBaseStruct {
+        self.dcx().emit_err(crate::diagnostics::CommaAfterBaseStruct {
             span: span.to(self.prev_token.span),
             comma: self.token.span,
         });
@@ -4040,7 +4009,8 @@ impl<'a> Parser<'a> {
         if !self.look_ahead(1, |t| t == close) && self.eat(exp!(DotDotDot)) {
             // recover from typo of `...`, suggest `..`
             let span = self.prev_token.span;
-            self.dcx().emit_err(diagnostics::MissingDotDot { token_span: span, sugg_span: span });
+            self.dcx()
+                .emit_err(crate::diagnostics::MissingDotDot { token_span: span, sugg_span: span });
             return true;
         }
         false
@@ -4053,7 +4023,7 @@ impl<'a> Parser<'a> {
         let label = format!("'{}", ident.name);
         let ident = Ident::new(Symbol::intern(&label), ident.span);
 
-        self.dcx().emit_err(diagnostics::ExpectedLabelFoundIdent {
+        self.dcx().emit_err(crate::diagnostics::ExpectedLabelFoundIdent {
             span: ident.span,
             start: ident.span.shrink_to_lo(),
         });
@@ -4080,7 +4050,7 @@ impl<'a> Parser<'a> {
                         || t == &token::CloseParen
                 });
             if is_wrong {
-                return Err(this.dcx().create_err(diagnostics::ExpectedStructField {
+                return Err(this.dcx().create_err(crate::diagnostics::ExpectedStructField {
                     span: this.look_ahead(1, |t| t.span),
                     ident_span: this.token.span,
                     token: pprust::token_to_string(&this.look_ahead(1, |t| *t)),
@@ -4121,20 +4091,12 @@ impl<'a> Parser<'a> {
             return;
         }
 
-        self.dcx().emit_err(diagnostics::EqFieldInit {
+        self.dcx().emit_err(crate::diagnostics::EqFieldInit {
             span: self.token.span,
             eq: field_name.span.shrink_to_hi().to(self.token.span),
         });
     }
 
-    fn err_dotdotdot_syntax(&self, span: Span) {
-        self.dcx().emit_err(diagnostics::DotDotDot { span });
-    }
-
-    fn err_larrow_operator(&self, span: Span) {
-        self.dcx().emit_err(diagnostics::LeftArrowOperator { span });
-    }
-
     fn mk_assign_op(&self, assign_op: AssignOp, lhs: Box<Expr>, rhs: Box<Expr>) -> ExprKind {
         ExprKind::AssignOp(assign_op, lhs, rhs)
     }
@@ -4282,9 +4244,9 @@ struct CondChecker<'a> {
     parser: &'a Parser<'a>,
     let_chains_policy: LetChainsPolicy,
     depth: u32,
-    forbid_let_reason: Option<diagnostics::ForbiddenLetReason>,
-    missing_let: Option<diagnostics::MaybeMissingLet>,
-    comparison: Option<diagnostics::MaybeComparison>,
+    forbid_let_reason: Option<crate::diagnostics::ForbiddenLetReason>,
+    missing_let: Option<crate::diagnostics::MaybeMissingLet>,
+    comparison: Option<crate::diagnostics::MaybeComparison>,
     found_incorrect_let_chain: Option<ErrorGuaranteed>,
 }
 
@@ -4311,12 +4273,13 @@ impl MutVisitor for CondChecker<'_> {
             ExprKind::Let(_, _, _, ref mut recovered @ Recovered::No) => {
                 if let Some(reason) = self.forbid_let_reason {
                     let error = match reason {
-                        diagnostics::ForbiddenLetReason::NotSupportedOr(or_span) => {
-                            self.parser.dcx().emit_err(diagnostics::OrInLetChain { span: or_span })
-                        }
+                        crate::diagnostics::ForbiddenLetReason::NotSupportedOr(or_span) => self
+                            .parser
+                            .dcx()
+                            .emit_err(crate::diagnostics::OrInLetChain { span: or_span }),
                         _ => {
                             let guar = self.parser.dcx().emit_err(
-                                diagnostics::ExpectedExpressionFoundLet {
+                                crate::diagnostics::ExpectedExpressionFoundLet {
                                     span,
                                     reason,
                                     missing_let: self.missing_let,
@@ -4336,7 +4299,9 @@ impl MutVisitor for CondChecker<'_> {
                         LetChainsPolicy::AlwaysAllowed => (),
                         LetChainsPolicy::EditionDependent { current_edition } => {
                             if !current_edition.at_least_rust_2024() || !span.at_least_rust_2024() {
-                                self.parser.dcx().emit_err(diagnostics::LetChainPre2024 { span });
+                                self.parser
+                                    .dcx()
+                                    .emit_err(crate::diagnostics::LetChainPre2024 { span });
                             }
                         }
                     }
@@ -4346,22 +4311,24 @@ impl MutVisitor for CondChecker<'_> {
                 mut_visit::walk_expr(self, e);
             }
             ExprKind::Binary(Spanned { node: BinOpKind::Or, span: or_span }, _, _)
-                if let None | Some(diagnostics::ForbiddenLetReason::NotSupportedOr(_)) =
+                if let None | Some(crate::diagnostics::ForbiddenLetReason::NotSupportedOr(_)) =
                     self.forbid_let_reason =>
             {
                 let forbid_let_reason = self.forbid_let_reason;
                 self.forbid_let_reason =
-                    Some(diagnostics::ForbiddenLetReason::NotSupportedOr(or_span));
+                    Some(crate::diagnostics::ForbiddenLetReason::NotSupportedOr(or_span));
                 mut_visit::walk_expr(self, e);
                 self.forbid_let_reason = forbid_let_reason;
             }
             ExprKind::Paren(ref inner)
-                if let None | Some(diagnostics::ForbiddenLetReason::NotSupportedParentheses(_)) =
+                if let None
+                | Some(crate::diagnostics::ForbiddenLetReason::NotSupportedParentheses(_)) =
                     self.forbid_let_reason =>
             {
                 let forbid_let_reason = self.forbid_let_reason;
-                self.forbid_let_reason =
-                    Some(diagnostics::ForbiddenLetReason::NotSupportedParentheses(inner.span));
+                self.forbid_let_reason = Some(
+                    crate::diagnostics::ForbiddenLetReason::NotSupportedParentheses(inner.span),
+                );
                 mut_visit::walk_expr(self, e);
                 self.forbid_let_reason = forbid_let_reason;
             }
@@ -4399,13 +4366,14 @@ impl MutVisitor for CondChecker<'_> {
                         if let Some(later_rhs) = find_let_some(rhs)
                             && depth > 0
                         {
-                            let guar =
-                                self.parser.dcx().emit_err(diagnostics::LetChainMissingLet {
+                            let guar = self.parser.dcx().emit_err(
+                                crate::diagnostics::LetChainMissingLet {
                                     span: lhs.span,
                                     label_span: expr_span,
                                     rhs_span: later_rhs.span,
                                     sug_span: lhs.span.shrink_to_lo(),
-                                });
+                                },
+                            );
 
                             self.found_incorrect_let_chain = Some(guar);
                         }
@@ -4413,7 +4381,8 @@ impl MutVisitor for CondChecker<'_> {
                 }
 
                 let forbid_let_reason = self.forbid_let_reason;
-                self.forbid_let_reason = Some(diagnostics::ForbiddenLetReason::OtherForbidden);
+                self.forbid_let_reason =
+                    Some(crate::diagnostics::ForbiddenLetReason::OtherForbidden);
                 let missing_let = self.missing_let;
                 if let ExprKind::Binary(_, _, rhs) = &lhs.kind
                     && let ExprKind::Path(_, _)
@@ -4422,10 +4391,11 @@ impl MutVisitor for CondChecker<'_> {
                     | ExprKind::Array(_) = rhs.kind
                 {
                     self.missing_let =
-                        Some(diagnostics::MaybeMissingLet { span: rhs.span.shrink_to_lo() });
+                        Some(crate::diagnostics::MaybeMissingLet { span: rhs.span.shrink_to_lo() });
                 }
                 let comparison = self.comparison;
-                self.comparison = Some(diagnostics::MaybeComparison { span: span.shrink_to_hi() });
+                self.comparison =
+                    Some(crate::diagnostics::MaybeComparison { span: span.shrink_to_hi() });
                 mut_visit::walk_expr(self, e);
                 self.forbid_let_reason = forbid_let_reason;
                 self.missing_let = missing_let;
@@ -4447,7 +4417,8 @@ impl MutVisitor for CondChecker<'_> {
             | ExprKind::Tup(_)
             | ExprKind::Paren(_) => {
                 let forbid_let_reason = self.forbid_let_reason;
-                self.forbid_let_reason = Some(diagnostics::ForbiddenLetReason::OtherForbidden);
+                self.forbid_let_reason =
+                    Some(crate::diagnostics::ForbiddenLetReason::OtherForbidden);
                 mut_visit::walk_expr(self, e);
                 self.forbid_let_reason = forbid_let_reason;
             }
@@ -4455,7 +4426,8 @@ impl MutVisitor for CondChecker<'_> {
             | ExprKind::Type(ref mut op, _)
             | ExprKind::UnsafeBinderCast(_, ref mut op, _) => {
                 let forbid_let_reason = self.forbid_let_reason;
-                self.forbid_let_reason = Some(diagnostics::ForbiddenLetReason::OtherForbidden);
+                self.forbid_let_reason =
+                    Some(crate::diagnostics::ForbiddenLetReason::OtherForbidden);
                 self.visit_expr(op);
                 self.forbid_let_reason = forbid_let_reason;
             }
diff --git a/compiler/rustc_parse/src/parser/expr/diagnostics.rs b/compiler/rustc_parse/src/parser/expr/diagnostics.rs
new file mode 100644
index 0000000000000..edbf844d5bbb0
--- /dev/null
+++ b/compiler/rustc_parse/src/parser/expr/diagnostics.rs
@@ -0,0 +1,80 @@
+use rustc_ast::util::parser::AssocOp;
+use rustc_ast::{BinOpKind, token};
+use rustc_span::Spanned;
+
+use crate::diagnostics;
+use crate::parser::Parser;
+
+impl<'a> Parser<'a> {
+    /// Reject `...` being used as an expression operator.
+    pub(super) fn reject_dotdotdot_expr_op(&self) {
+        if self.token == token::DotDotDot {
+            self.dcx().emit_err(diagnostics::DotDotDotExprOp { span: self.token.span });
+        }
+    }
+
+    /// Reject `<-` being used as an expression operator.
+    pub(super) fn reject_larrow_expr_op(&self) {
+        if self.token == token::LArrow {
+            self.dcx().emit_err(diagnostics::LArrowExprOp { span: self.token.span });
+        }
+    }
+
+    /// Recover from strict equality operators `===` and `!==` as found in e.g., JS and PHP.
+    pub(super) fn recover_from_strict_eq_op(&mut self, op: Spanned<AssocOp>) {
+        if let AssocOp::Binary(bop @ BinOpKind::Eq | bop @ BinOpKind::Ne) = op.node
+            && self.token == token::Eq
+            && self.prev_token.span.hi() == self.token.span.lo()
+        {
+            let sp = op.span.to(self.token.span);
+            let sugg = bop.as_str().into();
+            let invalid = format!("{sugg}=");
+            self.dcx().emit_err(diagnostics::InvalidComparisonOperator {
+                span: sp,
+                invalid: invalid.clone(),
+                sub: diagnostics::InvalidComparisonOperatorSub::Correctable {
+                    span: sp,
+                    invalid,
+                    correct: sugg,
+                },
+            });
+            self.bump();
+        }
+    }
+
+    /// Recover from inequality operator `<>` ("diamond") as found in e.g., PHP.
+    pub(super) fn recover_from_diamond_ne_op(&mut self, op: Spanned<AssocOp>) {
+        if op.node == AssocOp::Binary(BinOpKind::Lt)
+            && self.token == token::Gt
+            && self.prev_token.span.hi() == self.token.span.lo()
+        {
+            let sp = op.span.to(self.token.span);
+            self.dcx().emit_err(diagnostics::InvalidComparisonOperator {
+                span: sp,
+                invalid: "<>".into(),
+                sub: diagnostics::InvalidComparisonOperatorSub::Correctable {
+                    span: sp,
+                    invalid: "<>".into(),
+                    correct: "!=".into(),
+                },
+            });
+            self.bump();
+        }
+    }
+
+    /// Recover from comparison operator `<=>` ("spaceship") as found in e.g., C++.
+    pub(super) fn recover_from_spaceship_cmp_op(&mut self, op: Spanned<AssocOp>) {
+        if op.node == AssocOp::Binary(BinOpKind::Le)
+            && self.token == token::Gt
+            && self.prev_token.span.hi() == self.token.span.lo()
+        {
+            let sp = op.span.to(self.token.span);
+            self.dcx().emit_err(diagnostics::InvalidComparisonOperator {
+                span: sp,
+                invalid: "<=>".into(),
+                sub: diagnostics::InvalidComparisonOperatorSub::Spaceship(sp),
+            });
+            self.bump();
+        }
+    }
+}

From b945d684dc870d6cf0bbf058358c9f30e163f830 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Le=C3=B3n=20Orell=20Valerian=20Liehr?= <me@fmease.dev>
Date: Thu, 10 Sep 2026 13:25:49 +0200
Subject: [PATCH 21/28] Don't needlessly pass the operand through some recovery
 functions by value

These functions didn't actually modifiy the operand or return a new or
different expression. So essentially the "`fn(Box<Expr>) -> Box<Expr>`
part" was an identity function. Just change it to "fn(&Expr)".
---
 .../rustc_parse/src/parser/diagnostics.rs     | 26 ++++++++-----------
 compiler/rustc_parse/src/parser/expr.rs       |  7 ++---
 2 files changed, 15 insertions(+), 18 deletions(-)

diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs
index 40dbda2466de4..64f24b8216edd 100644
--- a/compiler/rustc_parse/src/parser/diagnostics.rs
+++ b/compiler/rustc_parse/src/parser/diagnostics.rs
@@ -1651,10 +1651,10 @@ impl<'a> Parser<'a> {
 
     pub(super) fn recover_from_prefix_increment(
         &mut self,
-        operand_expr: Box<Expr>,
+        operand_expr: &Expr,
         op_span: Span,
         start_stmt: bool,
-    ) -> PResult<'a, Box<Expr>> {
+    ) -> PResult<'a, ()> {
         let standalone = if start_stmt { IsStandalone::Standalone } else { IsStandalone::Subexpr };
         let kind = IncDecRecovery { standalone, op: IncOrDec::Inc, fixity: UnaryFixity::Pre };
         self.recover_from_inc_dec(operand_expr, kind, op_span)
@@ -1662,10 +1662,10 @@ impl<'a> Parser<'a> {
 
     pub(super) fn recover_from_postfix_increment(
         &mut self,
-        operand_expr: Box<Expr>,
+        operand_expr: &Expr,
         op_span: Span,
         start_stmt: bool,
-    ) -> PResult<'a, Box<Expr>> {
+    ) -> PResult<'a, ()> {
         let kind = IncDecRecovery {
             standalone: if start_stmt { IsStandalone::Standalone } else { IsStandalone::Subexpr },
             op: IncOrDec::Inc,
@@ -1676,10 +1676,10 @@ impl<'a> Parser<'a> {
 
     pub(super) fn recover_from_postfix_decrement(
         &mut self,
-        operand_expr: Box<Expr>,
+        operand_expr: &Expr,
         op_span: Span,
         start_stmt: bool,
-    ) -> PResult<'a, Box<Expr>> {
+    ) -> PResult<'a, ()> {
         let kind = IncDecRecovery {
             standalone: if start_stmt { IsStandalone::Standalone } else { IsStandalone::Subexpr },
             op: IncOrDec::Dec,
@@ -1690,22 +1690,16 @@ impl<'a> Parser<'a> {
 
     fn recover_from_inc_dec(
         &mut self,
-        base: Box<Expr>,
+        base: &Expr,
         kind: IncDecRecovery,
         op_span: Span,
-    ) -> PResult<'a, Box<Expr>> {
+    ) -> PResult<'a, ()> {
         let mut err = self.dcx().struct_span_err(
             op_span,
             format!("Rust has no {} {} operator", kind.fixity, kind.op.name()),
         );
         err.span_label(op_span, format!("not a valid {} operator", kind.fixity));
 
-        let help_base_case = |mut err: Diag<'_, ErrorGuaranteed>, base| {
-            err.help(format!("use `{}= 1` instead", kind.op.chr()));
-            err.emit();
-            Ok(base)
-        };
-
         // (pre, post)
         let spans = match kind.fixity {
             UnaryFixity::Pre => (op_span, base.span.shrink_to_hi()),
@@ -1718,7 +1712,9 @@ impl<'a> Parser<'a> {
             }
             IsStandalone::Subexpr => {
                 let Ok(base_src) = self.span_to_snippet(base.span) else {
-                    return help_base_case(err, base);
+                    err.help(format!("use `{}= 1` instead", kind.op.chr()));
+                    err.emit();
+                    return Ok(());
                 };
                 match kind.fixity {
                     UnaryFixity::Pre => {
diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
index a7bcb93d5b084..952c9a3fdf240 100644
--- a/compiler/rustc_parse/src/parser/expr.rs
+++ b/compiler/rustc_parse/src/parser/expr.rs
@@ -190,7 +190,7 @@ impl<'a> Parser<'a> {
                 let op_span = self.prev_token.span.to(self.token.span);
                 // Eat the second `+`
                 self.bump();
-                lhs = self.recover_from_postfix_increment(lhs, op_span, starts_stmt)?;
+                self.recover_from_postfix_increment(&lhs, op_span, starts_stmt)?;
                 continue;
             }
 
@@ -202,7 +202,7 @@ impl<'a> Parser<'a> {
                 let op_span = self.prev_token.span.to(self.token.span);
                 // Eat the second `-`
                 self.bump();
-                lhs = self.recover_from_postfix_decrement(lhs, op_span, starts_stmt)?;
+                self.recover_from_postfix_decrement(&lhs, op_span, starts_stmt)?;
                 continue;
             }
 
@@ -491,7 +491,8 @@ impl<'a> Parser<'a> {
                 this.bump();
 
                 let operand_expr = this.parse_expr_dot_or_call(attrs)?;
-                this.recover_from_prefix_increment(operand_expr, pre_span, starts_stmt)
+                this.recover_from_prefix_increment(&operand_expr, pre_span, starts_stmt)?;
+                Ok(operand_expr)
             }
             token::Ident(..)
                 if this.token.is_keyword(kw::Move)

From ceede0ba9c7dc2b2fb2fa68dc7b2b8b08e4372cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Le=C3=B3n=20Orell=20Valerian=20Liehr?= <me@fmease.dev>
Date: Sun, 6 Sep 2026 22:13:38 +0200
Subject: [PATCH 22/28] Remove odd special case of some parse error recovery
 functions

`recover_from_inc_dec` *always* returns a (fatal) `Err(_)` *except* if
the increment/decrement operator is a subexpression *and* the source of
the operand is not available in which case it emits the diagnostic and
returns `Ok(_)` (rendering it non-fatal).

This makes no sense whatsoever. For illustration purposes, listed below
are steps that would make us reach this case:

1. `rustc a.rs --crate-type=lib` where `a.rs` contains:
   `#[macro_export] macro_rules! m { () => { i++ } }`.
2. Move or remove `a.rs`
3. `rustc b.rs --edition 2018 --extern a -L.` where
   `b.rs` contains:
   `fn main() { (a::m!()); }`.

Just make the error unconditionally fatal and add a FIXME to make it non
fatal in the future which would allow us to report name resolution errors
and what not. However, since that would be slightly more involved and
represent a behavior change (in the error path), this is out of scope for
a mere cleanup commit like this one.
---
 .../rustc_parse/src/parser/diagnostics.rs     | 19 ++++++++++++-------
 compiler/rustc_parse/src/parser/expr.rs       | 13 +++++--------
 2 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs
index 64f24b8216edd..4aef934323bb3 100644
--- a/compiler/rustc_parse/src/parser/diagnostics.rs
+++ b/compiler/rustc_parse/src/parser/diagnostics.rs
@@ -1654,7 +1654,7 @@ impl<'a> Parser<'a> {
         operand_expr: &Expr,
         op_span: Span,
         start_stmt: bool,
-    ) -> PResult<'a, ()> {
+    ) -> Diag<'a> {
         let standalone = if start_stmt { IsStandalone::Standalone } else { IsStandalone::Subexpr };
         let kind = IncDecRecovery { standalone, op: IncOrDec::Inc, fixity: UnaryFixity::Pre };
         self.recover_from_inc_dec(operand_expr, kind, op_span)
@@ -1665,7 +1665,7 @@ impl<'a> Parser<'a> {
         operand_expr: &Expr,
         op_span: Span,
         start_stmt: bool,
-    ) -> PResult<'a, ()> {
+    ) -> Diag<'a> {
         let kind = IncDecRecovery {
             standalone: if start_stmt { IsStandalone::Standalone } else { IsStandalone::Subexpr },
             op: IncOrDec::Inc,
@@ -1679,7 +1679,7 @@ impl<'a> Parser<'a> {
         operand_expr: &Expr,
         op_span: Span,
         start_stmt: bool,
-    ) -> PResult<'a, ()> {
+    ) -> Diag<'a> {
         let kind = IncDecRecovery {
             standalone: if start_stmt { IsStandalone::Standalone } else { IsStandalone::Subexpr },
             op: IncOrDec::Dec,
@@ -1693,7 +1693,13 @@ impl<'a> Parser<'a> {
         base: &Expr,
         kind: IncDecRecovery,
         op_span: Span,
-    ) -> PResult<'a, ()> {
+    ) -> Diag<'a> {
+        // FIXME: Don't return an error diag, emit the diag here *and* return a new expr of the form
+        //        `$base += 1` / `$base -= 1` (taking `base: Expr` by value) for *proper* recovery.
+        //        (Just emitting the diag would be insufficient since callers would most likely just
+        //        use `$base` as the recovered AST node which would lead to annoying follow-up diags
+        //        like "variable doesn't need to be mutable" getting emitted in some cases.)
+
         let mut err = self.dcx().struct_span_err(
             op_span,
             format!("Rust has no {} {} operator", kind.fixity, kind.op.name()),
@@ -1713,8 +1719,7 @@ impl<'a> Parser<'a> {
             IsStandalone::Subexpr => {
                 let Ok(base_src) = self.span_to_snippet(base.span) else {
                     err.help(format!("use `{}= 1` instead", kind.op.chr()));
-                    err.emit();
-                    return Ok(());
+                    return err;
                 };
                 match kind.fixity {
                     UnaryFixity::Pre => {
@@ -1730,7 +1735,7 @@ impl<'a> Parser<'a> {
                 }
             }
         }
-        Err(err)
+        err
     }
 
     fn prefix_inc_dec_suggest(
diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
index 952c9a3fdf240..c179db3dbfa09 100644
--- a/compiler/rustc_parse/src/parser/expr.rs
+++ b/compiler/rustc_parse/src/parser/expr.rs
@@ -190,8 +190,7 @@ impl<'a> Parser<'a> {
                 let op_span = self.prev_token.span.to(self.token.span);
                 // Eat the second `+`
                 self.bump();
-                self.recover_from_postfix_increment(&lhs, op_span, starts_stmt)?;
-                continue;
+                return Err(self.recover_from_postfix_increment(&lhs, op_span, starts_stmt));
             }
 
             if self.prev_token == token::Minus
@@ -202,8 +201,7 @@ impl<'a> Parser<'a> {
                 let op_span = self.prev_token.span.to(self.token.span);
                 // Eat the second `-`
                 self.bump();
-                self.recover_from_postfix_decrement(&lhs, op_span, starts_stmt)?;
-                continue;
+                return Err(self.recover_from_postfix_decrement(&lhs, op_span, starts_stmt));
             }
 
             let op_span = op.span;
@@ -490,9 +488,8 @@ impl<'a> Parser<'a> {
                 this.bump();
                 this.bump();
 
-                let operand_expr = this.parse_expr_dot_or_call(attrs)?;
-                this.recover_from_prefix_increment(&operand_expr, pre_span, starts_stmt)?;
-                Ok(operand_expr)
+                let operand = this.parse_expr_dot_or_call(attrs)?;
+                return Err(this.recover_from_prefix_increment(&operand, pre_span, starts_stmt));
             }
             token::Ident(..)
                 if this.token.is_keyword(kw::Move)
@@ -503,7 +500,7 @@ impl<'a> Parser<'a> {
             token::Ident(..) if this.may_recover() && this.is_mistaken_not_ident_negation() => {
                 make_it!(this, attrs, |this, _| this.recover_not_expr(lo))
             }
-            _ => return this.parse_expr_dot_or_call(attrs),
+            _ => this.parse_expr_dot_or_call(attrs),
         }
     }
 

From 6c0ab88a2037fa75cc8d78d8b06a272f483e4710 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Le=C3=B3n=20Orell=20Valerian=20Liehr?= <me@fmease.dev>
Date: Sun, 6 Sep 2026 22:45:43 +0200
Subject: [PATCH 23/28] Dismantle bespoke diagnostic suggestion wrapper API

There's literally no upside to use it and only downsides:
It's not more concise, only adds code and obfuscates.
Its `MultiSugg::emit{,_verbose}` didn't even *emit* the diagnostic,
they merely *decorated* it!
---
 .../rustc_parse/src/parser/diagnostics.rs     | 109 ++++++------------
 1 file changed, 36 insertions(+), 73 deletions(-)

diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs
index 4aef934323bb3..3263fcacec498 100644
--- a/compiler/rustc_parse/src/parser/diagnostics.rs
+++ b/compiler/rustc_parse/src/parser/diagnostics.rs
@@ -211,22 +211,6 @@ fn find_similar_kw(lookup: Ident, candidates: &[Symbol]) -> Option<MisspelledKw>
     })
 }
 
-struct MultiSugg {
-    msg: String,
-    patches: Vec<(Span, String)>,
-    applicability: Applicability,
-}
-
-impl MultiSugg {
-    fn emit(self, err: &mut Diag<'_>) {
-        err.multipart_suggestion(self.msg, self.patches, self.applicability);
-    }
-
-    fn emit_verbose(self, err: &mut Diag<'_>) {
-        err.multipart_suggestion(self.msg, self.patches, self.applicability);
-    }
-}
-
 /// SnapshotParser is used to create a snapshot of the parser
 /// without causing duplicate errors being emitted when the `Parser`
 /// is dropped.
@@ -1706,15 +1690,23 @@ impl<'a> Parser<'a> {
         );
         err.span_label(op_span, format!("not a valid {} operator", kind.fixity));
 
-        // (pre, post)
-        let spans = match kind.fixity {
+        let (pre_span, post_span) = match kind.fixity {
             UnaryFixity::Pre => (op_span, base.span.shrink_to_hi()),
             UnaryFixity::Post => (base.span.shrink_to_lo(), op_span),
         };
 
         match kind.standalone {
             IsStandalone::Standalone => {
-                self.inc_dec_standalone_suggest(kind, spans).emit_verbose(&mut err)
+                let mut patches = Vec::new();
+                if !pre_span.is_empty() {
+                    patches.push((pre_span, String::new()));
+                }
+                patches.push((post_span, format!(" {}= 1", kind.op.chr())));
+                err.multipart_suggestion(
+                    format!("use `{}= 1` instead", kind.op.chr()),
+                    patches,
+                    Applicability::MachineApplicable,
+                );
             }
             IsStandalone::Subexpr => {
                 let Ok(base_src) = self.span_to_snippet(base.span) else {
@@ -1723,13 +1715,36 @@ impl<'a> Parser<'a> {
                 };
                 match kind.fixity {
                     UnaryFixity::Pre => {
-                        self.prefix_inc_dec_suggest(base_src, kind, spans).emit(&mut err)
+                        err.multipart_suggestion(
+                            format!("use `{}= 1` instead", kind.op.chr()),
+                            vec![
+                                (pre_span, "{ ".to_string()),
+                                (post_span, format!(" {}= 1; {} }}", kind.op.chr(), base_src)),
+                            ],
+                            Applicability::MachineApplicable,
+                        );
                     }
                     UnaryFixity::Post => {
                         // won't suggest since we can not handle the precedences
                         // for example: `a + b++` has been parsed (a + b)++ and we can not suggest here
                         if !matches!(base.kind, ExprKind::Binary(_, _, _)) {
-                            self.postfix_inc_dec_suggest(base_src, kind, spans).emit(&mut err)
+                            let tmp_var = if base_src.trim() == "tmp" { "tmp_" } else { "tmp" };
+                            err.multipart_suggestion(
+                                format!("use `{}= 1` instead", kind.op.chr()),
+                                vec![
+                                    (pre_span, format!("{{ let {tmp_var} = ")),
+                                    (
+                                        post_span,
+                                        format!(
+                                            "; {} {}= 1; {} }}",
+                                            base_src,
+                                            kind.op.chr(),
+                                            tmp_var
+                                        ),
+                                    ),
+                                ],
+                                Applicability::HasPlaceholders,
+                            );
                         }
                     }
                 }
@@ -1738,58 +1753,6 @@ impl<'a> Parser<'a> {
         err
     }
 
-    fn prefix_inc_dec_suggest(
-        &mut self,
-        base_src: String,
-        kind: IncDecRecovery,
-        (pre_span, post_span): (Span, Span),
-    ) -> MultiSugg {
-        MultiSugg {
-            msg: format!("use `{}= 1` instead", kind.op.chr()),
-            patches: vec![
-                (pre_span, "{ ".to_string()),
-                (post_span, format!(" {}= 1; {} }}", kind.op.chr(), base_src)),
-            ],
-            applicability: Applicability::MachineApplicable,
-        }
-    }
-
-    fn postfix_inc_dec_suggest(
-        &mut self,
-        base_src: String,
-        kind: IncDecRecovery,
-        (pre_span, post_span): (Span, Span),
-    ) -> MultiSugg {
-        let tmp_var = if base_src.trim() == "tmp" { "tmp_" } else { "tmp" };
-        MultiSugg {
-            msg: format!("use `{}= 1` instead", kind.op.chr()),
-            patches: vec![
-                (pre_span, format!("{{ let {tmp_var} = ")),
-                (post_span, format!("; {} {}= 1; {} }}", base_src, kind.op.chr(), tmp_var)),
-            ],
-            applicability: Applicability::HasPlaceholders,
-        }
-    }
-
-    fn inc_dec_standalone_suggest(
-        &mut self,
-        kind: IncDecRecovery,
-        (pre_span, post_span): (Span, Span),
-    ) -> MultiSugg {
-        let mut patches = Vec::new();
-
-        if !pre_span.is_empty() {
-            patches.push((pre_span, String::new()));
-        }
-
-        patches.push((post_span, format!(" {}= 1", kind.op.chr())));
-        MultiSugg {
-            msg: format!("use `{}= 1` instead", kind.op.chr()),
-            patches,
-            applicability: Applicability::MachineApplicable,
-        }
-    }
-
     /// Tries to recover from associated item paths like `[T]::AssocItem` / `(T, U)::AssocItem`.
     /// Attempts to convert the base expression/pattern/type into a type, parses the `::AssocItem`
     /// tail, and combines them into a `<Ty>::AssocItem` expression/pattern/type.

From ddb9380cde72a3abd88c8521e00c380451aa0e8f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Le=C3=B3n=20Orell=20Valerian=20Liehr?= <me@fmease.dev>
Date: Thu, 10 Sep 2026 10:37:14 +0200
Subject: [PATCH 24/28] Move parse error recovery from C-style inc/dec ops out
 of line

---
 compiler/rustc_parse/src/parser/expr.rs       | 23 +----------
 .../src/parser/expr/diagnostics.rs            | 38 ++++++++++++++++++-
 2 files changed, 39 insertions(+), 22 deletions(-)

diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
index c179db3dbfa09..1349af251e649 100644
--- a/compiler/rustc_parse/src/parser/expr.rs
+++ b/compiler/rustc_parse/src/parser/expr.rs
@@ -182,27 +182,8 @@ impl<'a> Parser<'a> {
             self.recover_from_strict_eq_op(op);
             self.recover_from_diamond_ne_op(op);
             self.recover_from_spaceship_cmp_op(op);
-
-            if self.prev_token == token::Plus
-                && self.token == token::Plus
-                && self.prev_token.span.between(self.token.span).is_empty()
-            {
-                let op_span = self.prev_token.span.to(self.token.span);
-                // Eat the second `+`
-                self.bump();
-                return Err(self.recover_from_postfix_increment(&lhs, op_span, starts_stmt));
-            }
-
-            if self.prev_token == token::Minus
-                && self.token == token::Minus
-                && self.prev_token.span.between(self.token.span).is_empty()
-                && !self.look_ahead(1, |tok| tok.can_begin_expr())
-            {
-                let op_span = self.prev_token.span.to(self.token.span);
-                // Eat the second `-`
-                self.bump();
-                return Err(self.recover_from_postfix_decrement(&lhs, op_span, starts_stmt));
-            }
+            self.recover_from_postfix_inc_op(&lhs, starts_stmt)?;
+            self.recover_from_postfix_dec_op(&lhs, starts_stmt)?;
 
             let op_span = op.span;
             let op = op.node;
diff --git a/compiler/rustc_parse/src/parser/expr/diagnostics.rs b/compiler/rustc_parse/src/parser/expr/diagnostics.rs
index edbf844d5bbb0..0008b576fdb4b 100644
--- a/compiler/rustc_parse/src/parser/expr/diagnostics.rs
+++ b/compiler/rustc_parse/src/parser/expr/diagnostics.rs
@@ -1,5 +1,6 @@
 use rustc_ast::util::parser::AssocOp;
-use rustc_ast::{BinOpKind, token};
+use rustc_ast::{BinOpKind, Expr, token};
+use rustc_errors::PResult;
 use rustc_span::Spanned;
 
 use crate::diagnostics;
@@ -77,4 +78,39 @@ impl<'a> Parser<'a> {
             self.bump();
         }
     }
+
+    /// Recover from postfix increment operator `++` as found in many C-style languages.
+    pub(super) fn recover_from_postfix_inc_op(
+        &mut self,
+        lhs: &Expr,
+        starts_stmt: bool,
+    ) -> PResult<'a, ()> {
+        if let (token::Plus, token::Plus) = (self.prev_token.kind, self.token.kind)
+            && self.prev_token.span.hi() == self.token.span.lo()
+        {
+            let op_span = self.prev_token.span.to(self.token.span);
+            self.bump(); // eat the second `+`
+            Err(self.recover_from_postfix_increment(lhs, op_span, starts_stmt))
+        } else {
+            Ok(())
+        }
+    }
+
+    /// Recover from postfix decrement operator `--` as found in many C-style languages.
+    pub(super) fn recover_from_postfix_dec_op(
+        &mut self,
+        lhs: &Expr,
+        starts_stmt: bool,
+    ) -> PResult<'a, ()> {
+        if let (token::Minus, token::Minus) = (self.prev_token.kind, self.token.kind)
+            && self.prev_token.span.hi() == self.token.span.lo()
+            && !self.look_ahead(1, |tok| tok.can_begin_expr())
+        {
+            let op_span = self.prev_token.span.to(self.token.span);
+            self.bump(); // eat the second `-`
+            Err(self.recover_from_postfix_decrement(lhs, op_span, starts_stmt))
+        } else {
+            Ok(())
+        }
+    }
 }

From 1adccf221ff9c6c507354395f255aa973c5a3693 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Le=C3=B3n=20Orell=20Valerian=20Liehr?= <me@fmease.dev>
Date: Thu, 10 Sep 2026 10:59:21 +0200
Subject: [PATCH 25/28] Inline fns & data types related to parse error recovery
 from C-style inc/dec ops

---
 .../rustc_parse/src/parser/diagnostics.rs     | 178 ------------------
 compiler/rustc_parse/src/parser/expr.rs       |   8 +-
 .../src/parser/expr/diagnostics.rs            | 104 +++++++++-
 3 files changed, 106 insertions(+), 184 deletions(-)

diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs
index 3263fcacec498..f5fa592585099 100644
--- a/compiler/rustc_parse/src/parser/diagnostics.rs
+++ b/compiler/rustc_parse/src/parser/diagnostics.rs
@@ -141,64 +141,6 @@ impl AttemptLocalParseRecovery {
     }
 }
 
-/// Information for emitting suggestions and recovering from
-/// C-style `i++`, `--i`, etc.
-#[derive(Debug, Copy, Clone)]
-struct IncDecRecovery {
-    /// Is this increment/decrement its own statement?
-    standalone: IsStandalone,
-    /// Is this an increment or decrement?
-    op: IncOrDec,
-    /// Is this pre- or postfix?
-    fixity: UnaryFixity,
-}
-
-/// Is an increment or decrement expression its own statement?
-#[derive(Debug, Copy, Clone)]
-enum IsStandalone {
-    /// It's standalone, i.e., its own statement.
-    Standalone,
-    /// It's a subexpression, i.e., *not* standalone.
-    Subexpr,
-}
-
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-enum IncOrDec {
-    Inc,
-    Dec,
-}
-
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-enum UnaryFixity {
-    Pre,
-    Post,
-}
-
-impl IncOrDec {
-    fn chr(&self) -> char {
-        match self {
-            Self::Inc => '+',
-            Self::Dec => '-',
-        }
-    }
-
-    fn name(&self) -> &'static str {
-        match self {
-            Self::Inc => "increment",
-            Self::Dec => "decrement",
-        }
-    }
-}
-
-impl std::fmt::Display for UnaryFixity {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            Self::Pre => write!(f, "prefix"),
-            Self::Post => write!(f, "postfix"),
-        }
-    }
-}
-
 /// Checks if the given `lookup` identifier is similar to any keyword symbol in `candidates`.
 ///
 /// This is a specialized version of [`Symbol::find_similar`] that constructs an error when a
@@ -1633,126 +1575,6 @@ impl<'a> Parser<'a> {
         Ok(())
     }
 
-    pub(super) fn recover_from_prefix_increment(
-        &mut self,
-        operand_expr: &Expr,
-        op_span: Span,
-        start_stmt: bool,
-    ) -> Diag<'a> {
-        let standalone = if start_stmt { IsStandalone::Standalone } else { IsStandalone::Subexpr };
-        let kind = IncDecRecovery { standalone, op: IncOrDec::Inc, fixity: UnaryFixity::Pre };
-        self.recover_from_inc_dec(operand_expr, kind, op_span)
-    }
-
-    pub(super) fn recover_from_postfix_increment(
-        &mut self,
-        operand_expr: &Expr,
-        op_span: Span,
-        start_stmt: bool,
-    ) -> Diag<'a> {
-        let kind = IncDecRecovery {
-            standalone: if start_stmt { IsStandalone::Standalone } else { IsStandalone::Subexpr },
-            op: IncOrDec::Inc,
-            fixity: UnaryFixity::Post,
-        };
-        self.recover_from_inc_dec(operand_expr, kind, op_span)
-    }
-
-    pub(super) fn recover_from_postfix_decrement(
-        &mut self,
-        operand_expr: &Expr,
-        op_span: Span,
-        start_stmt: bool,
-    ) -> Diag<'a> {
-        let kind = IncDecRecovery {
-            standalone: if start_stmt { IsStandalone::Standalone } else { IsStandalone::Subexpr },
-            op: IncOrDec::Dec,
-            fixity: UnaryFixity::Post,
-        };
-        self.recover_from_inc_dec(operand_expr, kind, op_span)
-    }
-
-    fn recover_from_inc_dec(
-        &mut self,
-        base: &Expr,
-        kind: IncDecRecovery,
-        op_span: Span,
-    ) -> Diag<'a> {
-        // FIXME: Don't return an error diag, emit the diag here *and* return a new expr of the form
-        //        `$base += 1` / `$base -= 1` (taking `base: Expr` by value) for *proper* recovery.
-        //        (Just emitting the diag would be insufficient since callers would most likely just
-        //        use `$base` as the recovered AST node which would lead to annoying follow-up diags
-        //        like "variable doesn't need to be mutable" getting emitted in some cases.)
-
-        let mut err = self.dcx().struct_span_err(
-            op_span,
-            format!("Rust has no {} {} operator", kind.fixity, kind.op.name()),
-        );
-        err.span_label(op_span, format!("not a valid {} operator", kind.fixity));
-
-        let (pre_span, post_span) = match kind.fixity {
-            UnaryFixity::Pre => (op_span, base.span.shrink_to_hi()),
-            UnaryFixity::Post => (base.span.shrink_to_lo(), op_span),
-        };
-
-        match kind.standalone {
-            IsStandalone::Standalone => {
-                let mut patches = Vec::new();
-                if !pre_span.is_empty() {
-                    patches.push((pre_span, String::new()));
-                }
-                patches.push((post_span, format!(" {}= 1", kind.op.chr())));
-                err.multipart_suggestion(
-                    format!("use `{}= 1` instead", kind.op.chr()),
-                    patches,
-                    Applicability::MachineApplicable,
-                );
-            }
-            IsStandalone::Subexpr => {
-                let Ok(base_src) = self.span_to_snippet(base.span) else {
-                    err.help(format!("use `{}= 1` instead", kind.op.chr()));
-                    return err;
-                };
-                match kind.fixity {
-                    UnaryFixity::Pre => {
-                        err.multipart_suggestion(
-                            format!("use `{}= 1` instead", kind.op.chr()),
-                            vec![
-                                (pre_span, "{ ".to_string()),
-                                (post_span, format!(" {}= 1; {} }}", kind.op.chr(), base_src)),
-                            ],
-                            Applicability::MachineApplicable,
-                        );
-                    }
-                    UnaryFixity::Post => {
-                        // won't suggest since we can not handle the precedences
-                        // for example: `a + b++` has been parsed (a + b)++ and we can not suggest here
-                        if !matches!(base.kind, ExprKind::Binary(_, _, _)) {
-                            let tmp_var = if base_src.trim() == "tmp" { "tmp_" } else { "tmp" };
-                            err.multipart_suggestion(
-                                format!("use `{}= 1` instead", kind.op.chr()),
-                                vec![
-                                    (pre_span, format!("{{ let {tmp_var} = ")),
-                                    (
-                                        post_span,
-                                        format!(
-                                            "; {} {}= 1; {} }}",
-                                            base_src,
-                                            kind.op.chr(),
-                                            tmp_var
-                                        ),
-                                    ),
-                                ],
-                                Applicability::HasPlaceholders,
-                            );
-                        }
-                    }
-                }
-            }
-        }
-        err
-    }
-
     /// Tries to recover from associated item paths like `[T]::AssocItem` / `(T, U)::AssocItem`.
     /// Attempts to convert the base expression/pattern/type into a type, parses the `::AssocItem`
     /// tail, and combines them into a `<Ty>::AssocItem` expression/pattern/type.
diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
index 1349af251e649..54e10d05f1140 100644
--- a/compiler/rustc_parse/src/parser/expr.rs
+++ b/compiler/rustc_parse/src/parser/expr.rs
@@ -470,7 +470,13 @@ impl<'a> Parser<'a> {
                 this.bump();
 
                 let operand = this.parse_expr_dot_or_call(attrs)?;
-                return Err(this.recover_from_prefix_increment(&operand, pre_span, starts_stmt));
+                return Err(this.report_inc_dec_op(
+                    &operand,
+                    starts_stmt,
+                    diagnostics::IncOrDec::Inc,
+                    diagnostics::UnaryFixity::Pre,
+                    pre_span,
+                ));
             }
             token::Ident(..)
                 if this.token.is_keyword(kw::Move)
diff --git a/compiler/rustc_parse/src/parser/expr/diagnostics.rs b/compiler/rustc_parse/src/parser/expr/diagnostics.rs
index 0008b576fdb4b..18dd6dba13eb5 100644
--- a/compiler/rustc_parse/src/parser/expr/diagnostics.rs
+++ b/compiler/rustc_parse/src/parser/expr/diagnostics.rs
@@ -1,7 +1,7 @@
 use rustc_ast::util::parser::AssocOp;
-use rustc_ast::{BinOpKind, Expr, token};
-use rustc_errors::PResult;
-use rustc_span::Spanned;
+use rustc_ast::{BinOpKind, Expr, ExprKind, token};
+use rustc_errors::{Applicability, Diag, PResult};
+use rustc_span::{Span, Spanned};
 
 use crate::diagnostics;
 use crate::parser::Parser;
@@ -90,7 +90,7 @@ impl<'a> Parser<'a> {
         {
             let op_span = self.prev_token.span.to(self.token.span);
             self.bump(); // eat the second `+`
-            Err(self.recover_from_postfix_increment(lhs, op_span, starts_stmt))
+            Err(self.report_inc_dec_op(lhs, starts_stmt, IncOrDec::Inc, UnaryFixity::Post, op_span))
         } else {
             Ok(())
         }
@@ -108,9 +108,103 @@ impl<'a> Parser<'a> {
         {
             let op_span = self.prev_token.span.to(self.token.span);
             self.bump(); // eat the second `-`
-            Err(self.recover_from_postfix_decrement(lhs, op_span, starts_stmt))
+            Err(self.report_inc_dec_op(lhs, starts_stmt, IncOrDec::Dec, UnaryFixity::Post, op_span))
         } else {
             Ok(())
         }
     }
+
+    /// Report increment operator `++` & decrement operator `--` as found in many C-style languages.
+    pub(super) fn report_inc_dec_op(
+        &mut self,
+        base: &Expr,
+        starts_stmt: bool,
+        op: IncOrDec,
+        fixity: UnaryFixity,
+        op_span: Span,
+    ) -> Diag<'a> {
+        // FIXME: Don't return an error diag, emit the diag here *and* return a new expr of the form
+        //        `$base += 1` / `$base -= 1` (taking `base: Expr` by value) for *proper* recovery.
+        //        (Just emitting the diag would be insufficient since callers would most likely just
+        //        use `$base` as the recovered AST node which would lead to annoying follow-up diags
+        //        like "variable doesn't need to be mutable" getting emitted in some cases.)
+
+        let mut err = {
+            let fixity = match fixity {
+                UnaryFixity::Pre => "prefix",
+                UnaryFixity::Post => "postfix",
+            };
+            let op = match op {
+                IncOrDec::Inc => "increment",
+                IncOrDec::Dec => "decrement",
+            };
+            self.dcx()
+                .struct_span_err(op_span, format!("Rust has no {fixity} {op} operator"))
+                .with_span_label(op_span, format!("not a valid {fixity} operator"))
+        };
+
+        let op = match op {
+            IncOrDec::Inc => "+= 1",
+            IncOrDec::Dec => "-= 1",
+        };
+        let (pre_span, post_span) = match fixity {
+            UnaryFixity::Pre => (op_span, base.span.shrink_to_hi()),
+            UnaryFixity::Post => (base.span.shrink_to_lo(), op_span),
+        };
+
+        if starts_stmt {
+            let mut patches = Vec::new();
+            if !pre_span.is_empty() {
+                patches.push((pre_span, String::new()));
+            }
+            patches.push((post_span, format!(" {op}")));
+            err.multipart_suggestion(
+                format!("use `{op}` instead"),
+                patches,
+                Applicability::MachineApplicable,
+            );
+        } else {
+            let Ok(base_src) = self.span_to_snippet(base.span) else {
+                err.help(format!("use `{op}` instead"));
+                return err;
+            };
+            match fixity {
+                UnaryFixity::Pre => {
+                    err.multipart_suggestion(
+                        format!("use `{op}` instead"),
+                        vec![(pre_span, "{ ".into()), (post_span, format!(" {op}; {base_src} }}"))],
+                        Applicability::MachineApplicable,
+                    );
+                }
+                UnaryFixity::Post => {
+                    // won't suggest since we can not handle the precedences
+                    // for example: `a + b++` has been parsed (a + b)++ and we can not suggest here
+                    if !matches!(base.kind, ExprKind::Binary(..)) {
+                        let tmp_var = if base_src.trim() == "tmp" { "tmp_" } else { "tmp" };
+                        err.multipart_suggestion(
+                            format!("use `{op}` instead"),
+                            vec![
+                                (pre_span, format!("{{ let {tmp_var} = ")),
+                                (post_span, format!("; {base_src} {op}; {tmp_var} }}")),
+                            ],
+                            Applicability::HasPlaceholders,
+                        );
+                    }
+                }
+            }
+        }
+        err
+    }
+}
+
+#[derive(Copy, Clone)]
+pub(super) enum IncOrDec {
+    Inc,
+    Dec,
+}
+
+#[derive(Copy, Clone)]
+pub(super) enum UnaryFixity {
+    Pre,
+    Post,
 }

From d85d3f55051e0d88237c0f4c0b4070f73d5ef837 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Le=C3=B3n=20Orell=20Valerian=20Liehr?= <me@fmease.dev>
Date: Thu, 10 Sep 2026 12:05:49 +0200
Subject: [PATCH 26/28] Refactor the way we finish parsing expr ops

1. Remove unnecessary rebindings (`op_span` and `op = op.node`)
2. Remove binding `cur_op_span` as it's equal to `op.span`
3. Merge two `match`es on `op.node` into one to make the control flow
   more obvious and to render everything more legible. Moreover,
   it allows us to drop an ungly `unreachable!()`
---
 compiler/rustc_parse/src/parser/expr.rs | 56 ++++++++++++-------------
 1 file changed, 27 insertions(+), 29 deletions(-)

diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
index 54e10d05f1140..336af55a4e904 100644
--- a/compiler/rustc_parse/src/parser/expr.rs
+++ b/compiler/rustc_parse/src/parser/expr.rs
@@ -152,7 +152,6 @@ impl<'a> Parser<'a> {
         self.expected_token_types.insert(TokenType::Operator);
         while let Some(op) = self.check_assoc_op() {
             let lhs_span = self.interpolated_or_expr_span(&lhs);
-            let cur_op_span = self.token.span;
             let restrictions = if op.node.is_assign_like() {
                 self.restrictions & Restrictions::NO_STRUCT_LITERAL
             } else {
@@ -185,42 +184,41 @@ impl<'a> Parser<'a> {
             self.recover_from_postfix_inc_op(&lhs, starts_stmt)?;
             self.recover_from_postfix_dec_op(&lhs, starts_stmt)?;
 
-            let op_span = op.span;
-            let op = op.node;
-            // Special cases:
-            if op == AssocOp::Cast {
-                lhs = self.parse_assoc_op_cast(lhs, lhs_span, op_span, ExprKind::Cast)?;
-                continue;
-            } else if let AssocOp::Range(limits) = op {
-                // If we didn't have to handle `x..`/`x..=`, it would be pretty easy to
-                // generalise it to the Fixity::None code.
-                lhs = self.parse_expr_range(prec, lhs, limits, cur_op_span)?;
-                break;
-            }
-
-            let min_prec = match op.fixity() {
+            let min_prec = match op.node.fixity() {
                 Fixity::Right => Bound::Included(prec),
                 Fixity::Left | Fixity::None => Bound::Excluded(prec),
             };
-            let rhs = self.with_res(restrictions - Restrictions::STMT_EXPR, |this| {
-                this.parse_expr_assoc(min_prec)
-            })?;
 
-            let span = self.mk_expr_sp(&lhs, lhs_span, op_span, rhs.span);
-            lhs = match op {
+            let finish_parsing_bin_op = |this: &mut Self| {
+                let rhs = this.with_res(restrictions - Restrictions::STMT_EXPR, |this| {
+                    this.parse_expr_assoc(min_prec)
+                })?;
+                let span = this.mk_expr_sp(&lhs, lhs_span, op.span, rhs.span);
+                Ok((rhs, span))
+            };
+
+            lhs = match op.node {
                 AssocOp::Binary(ast_op) => {
-                    let binary = self.mk_binary(respan(cur_op_span, ast_op), lhs, rhs);
-                    self.mk_expr(span, binary)
+                    let (rhs, span) = finish_parsing_bin_op(self)?;
+                    self.mk_expr(span, self.mk_binary(respan(op.span, ast_op), lhs, rhs))
                 }
-                AssocOp::Assign => self.mk_expr(span, ExprKind::Assign(lhs, rhs, cur_op_span)),
                 AssocOp::AssignOp(aop) => {
-                    let aopexpr = self.mk_assign_op(respan(cur_op_span, aop), lhs, rhs);
-                    self.mk_expr(span, aopexpr)
+                    let (rhs, span) = finish_parsing_bin_op(self)?;
+                    self.mk_expr(span, self.mk_assign_op(respan(op.span, aop), lhs, rhs))
+                }
+                AssocOp::Assign => {
+                    let (rhs, span) = finish_parsing_bin_op(self)?;
+                    self.mk_expr(span, ExprKind::Assign(lhs, rhs, op.span))
                 }
-                AssocOp::Cast | AssocOp::Range(_) => {
-                    self.dcx().span_bug(span, "AssocOp should have been handled by special case")
+                AssocOp::Cast => {
+                    self.parse_assoc_op_cast(lhs, lhs_span, op.span, ExprKind::Cast)?
                 }
+                AssocOp::Range(limits) => self.parse_expr_range(min_prec, lhs, limits, op.span)?,
             };
+
+            if let AssocOp::Range(_) = op.node {
+                break;
+            }
         }
 
         Ok((lhs, parsed_something))
@@ -338,7 +336,7 @@ impl<'a> Parser<'a> {
     /// The other two variants are handled in `parse_prefix_range_expr` below.
     fn parse_expr_range(
         &mut self,
-        prec: ExprPrecedence,
+        min_prec: Bound<ExprPrecedence>,
         lhs: Box<Expr>,
         limits: RangeLimits,
         cur_op_span: Span,
@@ -346,7 +344,7 @@ impl<'a> Parser<'a> {
         let rhs = if self.is_at_start_of_range_notation_rhs() {
             let maybe_lt = self.token;
             Some(
-                self.parse_expr_assoc(Bound::Excluded(prec))
+                self.parse_expr_assoc(min_prec)
                     .map_err(|err| self.maybe_err_dotdotlt_syntax(maybe_lt, err))?,
             )
         } else {

From f0ae097b364ffd9b30f908aeddc0b9eb040464dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Le=C3=B3n=20Orell=20Valerian=20Liehr?= <me@fmease.dev>
Date: Tue, 25 Aug 2026 16:30:11 +0200
Subject: [PATCH 27/28] Refactor `check_assoc_op` to make it more legible

---
 compiler/rustc_parse/src/diagnostics.rs       |  2 +-
 compiler/rustc_parse/src/parser/expr.rs       | 80 +++++++------------
 .../src/parser/expr/diagnostics.rs            | 25 +++++-
 3 files changed, 53 insertions(+), 54 deletions(-)

diff --git a/compiler/rustc_parse/src/diagnostics.rs b/compiler/rustc_parse/src/diagnostics.rs
index 78240b0ee891b..4f3c702c77ef9 100644
--- a/compiler/rustc_parse/src/diagnostics.rs
+++ b/compiler/rustc_parse/src/diagnostics.rs
@@ -257,7 +257,7 @@ pub(crate) enum InvalidComparisonOperatorSub {
 pub(crate) struct InvalidLogicalOperator {
     #[primary_span]
     pub span: Span,
-    pub incorrect: String,
+    pub incorrect: Symbol,
     #[subdiagnostic]
     pub sub: InvalidLogicalOperatorSub,
 }
diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
index 336af55a4e904..148e49d803875 100644
--- a/compiler/rustc_parse/src/parser/expr.rs
+++ b/compiler/rustc_parse/src/parser/expr.rs
@@ -272,59 +272,35 @@ impl<'a> Parser<'a> {
 
     /// Possibly translate the current token to an associative operator.
     /// The method does not advance the current token.
-    ///
-    /// Also performs recovery for `and` / `or` which are mistaken for `&&` and `||` respectively.
     pub(super) fn check_assoc_op(&self) -> Option<Spanned<AssocOp>> {
-        let (op, span) = match (AssocOp::from_token(&self.token), self.token.ident()) {
-            // When parsing const expressions, stop parsing when encountering `>`.
-            (
-                Some(
-                    AssocOp::Binary(BinOpKind::Shr | BinOpKind::Gt | BinOpKind::Ge)
-                    | AssocOp::AssignOp(AssignOpKind::ShrAssign),
-                ),
-                _,
-            ) if self.restrictions.contains(Restrictions::CONST_EXPR) => {
-                return None;
-            }
-            // When recovering patterns as expressions, stop parsing when encountering an
-            // assignment `=`, an alternative `|`, or a range `..`.
-            (
-                Some(
-                    AssocOp::Assign
-                    | AssocOp::AssignOp(_)
-                    | AssocOp::Binary(BinOpKind::BitOr)
-                    | AssocOp::Range(_),
-                ),
-                _,
-            ) if self.restrictions.contains(Restrictions::IS_PAT) => {
-                return None;
-            }
-            (Some(op), _) => (op, self.token.span),
-            (None, Some((Ident { name: sym::and, span }, IdentIsRaw::No)))
-                if self.may_recover() =>
-            {
-                self.dcx().emit_err(crate::diagnostics::InvalidLogicalOperator {
-                    span: self.token.span,
-                    incorrect: "and".into(),
-                    sub: crate::diagnostics::InvalidLogicalOperatorSub::Conjunction(
-                        self.token.span,
-                    ),
-                });
-                (AssocOp::Binary(BinOpKind::And), span)
-            }
-            (None, Some((Ident { name: sym::or, span }, IdentIsRaw::No))) if self.may_recover() => {
-                self.dcx().emit_err(crate::diagnostics::InvalidLogicalOperator {
-                    span: self.token.span,
-                    incorrect: "or".into(),
-                    sub: crate::diagnostics::InvalidLogicalOperatorSub::Disjunction(
-                        self.token.span,
-                    ),
-                });
-                (AssocOp::Binary(BinOpKind::Or), span)
-            }
-            _ => return None,
-        };
-        Some(respan(span, op))
+        let op = AssocOp::from_token(&self.token);
+
+        // When parsing const expressions, stop parsing when encountering `>`.
+        if self.restrictions.contains(Restrictions::CONST_EXPR)
+            && let Some(op) = op
+            && let AssocOp::Binary(BinOpKind::Shr | BinOpKind::Gt | BinOpKind::Ge)
+            | AssocOp::AssignOp(AssignOpKind::ShrAssign) = op
+        {
+            return None;
+        }
+
+        // When recovering patterns as expressions, stop parsing when encountering an
+        // assignment `=`, an alternative `|`, or a range `..`.
+        if self.restrictions.contains(Restrictions::IS_PAT)
+            && let Some(op) = op
+            && let AssocOp::Assign
+            | AssocOp::AssignOp(_)
+            | AssocOp::Binary(BinOpKind::BitOr)
+            | AssocOp::Range(_) = op
+        {
+            return None;
+        }
+
+        if let Some(op) = op {
+            return Some(respan(self.token.span, op));
+        }
+
+        self.recover_from_alpha_logic_op()
     }
 
     /// Checks if this expression is a successfully parsed statement.
diff --git a/compiler/rustc_parse/src/parser/expr/diagnostics.rs b/compiler/rustc_parse/src/parser/expr/diagnostics.rs
index 18dd6dba13eb5..4b9e288f3ec1b 100644
--- a/compiler/rustc_parse/src/parser/expr/diagnostics.rs
+++ b/compiler/rustc_parse/src/parser/expr/diagnostics.rs
@@ -1,12 +1,35 @@
 use rustc_ast::util::parser::AssocOp;
 use rustc_ast::{BinOpKind, Expr, ExprKind, token};
 use rustc_errors::{Applicability, Diag, PResult};
-use rustc_span::{Span, Spanned};
+use rustc_span::{Span, Spanned, respan, sym};
 
 use crate::diagnostics;
 use crate::parser::Parser;
 
 impl<'a> Parser<'a> {
+    /// Recover from alphabetic logic operators `and` and `or` as found in e.g., Python and PHP.
+    pub(super) fn recover_from_alpha_logic_op(&self) -> Option<Spanned<AssocOp>> {
+        if self.may_recover()
+            && let Some((ident, token::IdentIsRaw::No)) = self.token.ident()
+        {
+            let (op, sub): (_, fn(_) -> _) = match ident.name {
+                sym::and => (BinOpKind::And, diagnostics::InvalidLogicalOperatorSub::Conjunction),
+                sym::or => (BinOpKind::Or, diagnostics::InvalidLogicalOperatorSub::Disjunction),
+                _ => return None,
+            };
+
+            self.dcx().emit_err(diagnostics::InvalidLogicalOperator {
+                span: self.token.span,
+                incorrect: ident.name,
+                sub: sub(self.token.span),
+            });
+
+            Some(respan(self.token.span, AssocOp::Binary(op)))
+        } else {
+            None
+        }
+    }
+
     /// Reject `...` being used as an expression operator.
     pub(super) fn reject_dotdotdot_expr_op(&self) {
         if self.token == token::DotDotDot {

From 28158721a2f166c635a091b88f09650e3f28da57 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Le=C3=B3n=20Orell=20Valerian=20Liehr?= <me@fmease.dev>
Date: Fri, 11 Sep 2026 10:24:51 +0200
Subject: [PATCH 28/28] Don't mistake `<->` for `<>`

Previously we would check if the current operator was `Binary(Lt)` and
the current token was `>` to determine if we're looking at `<>`.

However, since `AssocOp::from_token` also treats `<-` as `Binary(Lt)`
for better error recovery, the condition would also hold for `<->`
(`<-`, `>`) which is not what we want. E.g., given `1 <-> 2` we would
previously emit diagnostic "invalid comparison operator `<>`".

---

Also update `recover_from_spaceship_cmp_op` to do something similar --
not to fix anything but simply to eliminate param `op: Spanned<AssocOp>`.
---
 compiler/rustc_parse/src/parser/expr.rs            |  4 ++--
 .../rustc_parse/src/parser/expr/diagnostics.rs     | 14 ++++++--------
 2 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
index 148e49d803875..58e98a64b5e41 100644
--- a/compiler/rustc_parse/src/parser/expr.rs
+++ b/compiler/rustc_parse/src/parser/expr.rs
@@ -179,8 +179,8 @@ impl<'a> Parser<'a> {
             }
 
             self.recover_from_strict_eq_op(op);
-            self.recover_from_diamond_ne_op(op);
-            self.recover_from_spaceship_cmp_op(op);
+            self.recover_from_diamond_ne_op();
+            self.recover_from_spaceship_cmp_op();
             self.recover_from_postfix_inc_op(&lhs, starts_stmt)?;
             self.recover_from_postfix_dec_op(&lhs, starts_stmt)?;
 
diff --git a/compiler/rustc_parse/src/parser/expr/diagnostics.rs b/compiler/rustc_parse/src/parser/expr/diagnostics.rs
index 4b9e288f3ec1b..707ae5d34bc75 100644
--- a/compiler/rustc_parse/src/parser/expr/diagnostics.rs
+++ b/compiler/rustc_parse/src/parser/expr/diagnostics.rs
@@ -67,12 +67,11 @@ impl<'a> Parser<'a> {
     }
 
     /// Recover from inequality operator `<>` ("diamond") as found in e.g., PHP.
-    pub(super) fn recover_from_diamond_ne_op(&mut self, op: Spanned<AssocOp>) {
-        if op.node == AssocOp::Binary(BinOpKind::Lt)
-            && self.token == token::Gt
+    pub(super) fn recover_from_diamond_ne_op(&mut self) {
+        if let (token::Lt, token::Gt) = (self.prev_token.kind, self.token.kind)
             && self.prev_token.span.hi() == self.token.span.lo()
         {
-            let sp = op.span.to(self.token.span);
+            let sp = self.prev_token.span.to(self.token.span);
             self.dcx().emit_err(diagnostics::InvalidComparisonOperator {
                 span: sp,
                 invalid: "<>".into(),
@@ -87,12 +86,11 @@ impl<'a> Parser<'a> {
     }
 
     /// Recover from comparison operator `<=>` ("spaceship") as found in e.g., C++.
-    pub(super) fn recover_from_spaceship_cmp_op(&mut self, op: Spanned<AssocOp>) {
-        if op.node == AssocOp::Binary(BinOpKind::Le)
-            && self.token == token::Gt
+    pub(super) fn recover_from_spaceship_cmp_op(&mut self) {
+        if let (token::Le, token::Gt) = (self.prev_token.kind, self.token.kind)
             && self.prev_token.span.hi() == self.token.span.lo()
         {
-            let sp = op.span.to(self.token.span);
+            let sp = self.prev_token.span.to(self.token.span);
             self.dcx().emit_err(diagnostics::InvalidComparisonOperator {
                 span: sp,
                 invalid: "<=>".into(),