Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 2 additions & 5 deletions compiler/rustc_codegen_llvm/src/back/llvm_backend.rs
Original file line number Diff line number Diff line change
Expand Up @@ -208,11 +208,12 @@ impl CodegenBackend for LlvmCodegenBackend {
// Intrinsics whose fallback body will not be used by the LLVM backend.
let replaced_intrinsics = {
#[rustfmt::skip]
let mut will_not_use_fallback = vec![
let will_not_use_fallback = vec![
// These are mapped to LLVM intrinsics instead.
sym::unchecked_funnel_shl,
sym::unchecked_funnel_shr,
sym::carrying_mul_add,
sym::carryless_mul,
sym::integer_max,
sym::integer_min,

Expand All @@ -235,10 +236,6 @@ impl CodegenBackend for LlvmCodegenBackend {
sym::copysignf16, sym::copysignf32, sym::copysignf64, sym::copysignf128,
];

if llvm_util::get_version() >= (22, 0, 0) {
will_not_use_fallback.push(sym::carryless_mul);
}

will_not_use_fallback
};

Expand Down
16 changes: 5 additions & 11 deletions compiler/rustc_codegen_llvm/src/builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -919,9 +919,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
self.set_metadata_node(store, llvm::MD_nontemporal, &[one]);
}
}
if flags.contains(MemFlags::CAPTURES_READ_ONLY)
&& crate::llvm_util::get_version() >= (22, 0, 0)
{
if flags.contains(MemFlags::CAPTURES_READ_ONLY) {
assert!(
self.type_kind(self.val_ty(val)) == TypeKind::Pointer,
"CAPTURED_READ_ONLY is only supported on pointer stores"
Expand Down Expand Up @@ -1903,14 +1901,10 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
return;
}

if crate::llvm_util::get_version() >= (22, 0, 0) {
// LLVM 22 requires the lifetime intrinsic to act directly on the alloca,
// there can't be an addrspacecast in between.
let ptr = unsafe { llvm::LLVMRustStripPointerCasts(ptr) };
self.call_intrinsic(intrinsic, &[self.val_ty(ptr)], &[ptr]);
} else {
self.call_intrinsic(intrinsic, &[self.val_ty(ptr)], &[self.cx.const_u64(size), ptr]);
}
// LLVM 22 requires the lifetime intrinsic to act directly on the alloca,
// there can't be an addrspacecast in between.
let ptr = unsafe { llvm::LLVMRustStripPointerCasts(ptr) };
self.call_intrinsic(intrinsic, &[self.val_ty(ptr)], &[ptr]);
}
}
impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
Expand Down
27 changes: 0 additions & 27 deletions compiler/rustc_codegen_llvm/src/context.rs
Original file line number Diff line number Diff line change
Expand Up @@ -195,33 +195,6 @@ pub(crate) unsafe fn create_module<'ll>(
let mut target_data_layout = sess.target.data_layout.to_string();
let llvm_version = llvm_util::get_version();

if llvm_version < (22, 0, 0) {
if sess.target.arch == Arch::Avr {
// LLVM 22.0 updated the default layout on avr: https://github.com/llvm/llvm-project/pull/153010
target_data_layout = target_data_layout.replace("n8:16", "n8")
}
if sess.target.arch == Arch::Nvptx64 {
// LLVM 22 updated the NVPTX layout to indicate 256-bit vector load/store: https://github.com/llvm/llvm-project/pull/155198
target_data_layout = target_data_layout.replace("-i256:256", "");
}
if sess.target.arch == Arch::PowerPC64 {
// LLVM 22 updated the ABI alignment for double on AIX: https://github.com/llvm/llvm-project/pull/144673
target_data_layout = target_data_layout.replace("-f64:32:64", "");

// LLVM 22 fixed the data layout calculation for targets that default to ELFv1
// when the ABI is set to ELFv2. With LLVM 21, the ELFv1 datalayout must be used,
// which will overalign function entries.
// https://github.com/llvm/llvm-project/pull/149725
if sess.target.llvm_target == "powerpc64-unknown-linux-gnu" {
target_data_layout = target_data_layout.replace("-Fn32", "-Fi64");
}
}
if sess.target.arch == Arch::AmdGpu {
// LLVM 22 specified ELF mangling in the amdgpu data layout:
// https://github.com/llvm/llvm-project/pull/163011
target_data_layout = target_data_layout.replace("-m:e", "");
}
}
if llvm_version < (23, 0, 0) {
if sess.target.arch == Arch::S390x {
// LLVM 23 updated the s390x layout to specify the stack alignment: https://github.com/llvm/llvm-project/pull/176041
Expand Down
147 changes: 49 additions & 98 deletions compiler/rustc_codegen_llvm/src/intrinsic.rs
Original file line number Diff line number Diff line change
Expand Up @@ -192,17 +192,14 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
let simple = call_simple_intrinsic(self, name, args);
let llval = match name {
_ if simple.is_some() => simple.unwrap(),
// Need at least LLVM 22 for `min/maximumnum` to not crash LLVM.
sym::minimum_number_nsz_f16
| sym::minimum_number_nsz_f32
| sym::minimum_number_nsz_f64
| sym::minimum_number_nsz_f128
| sym::maximum_number_nsz_f16
| sym::maximum_number_nsz_f32
| sym::maximum_number_nsz_f64
| sym::maximum_number_nsz_f128
if llvm_version >= (22, 0, 0) =>
{
| sym::maximum_number_nsz_f128 => {
let intrinsic_name = if name.as_str().starts_with("min") {
"llvm.minimumnum"
} else {
Expand Down Expand Up @@ -457,33 +454,14 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
pair
}

// FIXME move into the branch below when LLVM 22 is the lowest version we support.
sym::carryless_mul if llvm_version >= (22, 0, 0) => {
let ty = args[0].layout.ty;
if !ty.is_integral() {
let err = tcx.dcx().emit_err(InvalidMonomorphization::BasicIntegerType {
span,
name,
ty,
});
return IntrinsicResult::Err(err);
}
let (size, _) = ty.int_size_and_signed(self.tcx);
let width = size.bits();
let llty = self.type_ix(width);

let lhs = args[0].immediate();
let rhs = args[1].immediate();
self.call_intrinsic("llvm.clmul", &[llty], &[lhs, rhs])
}

sym::ctlz
| sym::ctlz_nonzero
| sym::cttz
| sym::cttz_nonzero
| sym::ctpop
| sym::bswap
| sym::bitreverse
| sym::carryless_mul
| sym::integer_max
| sym::integer_min
| sym::saturating_add
Expand Down Expand Up @@ -530,6 +508,11 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
sym::bitreverse => {
self.call_intrinsic("llvm.bitreverse", &[llty], &[args[0].immediate()])
}
sym::carryless_mul => {
let lhs = args[0].immediate();
let rhs = args[1].immediate();
self.call_intrinsic("llvm.clmul", &[llty], &[lhs, rhs])
}
sym::integer_min | sym::integer_max => {
let lhs = args[0].immediate();
let rhs = args[1].immediate();
Expand Down Expand Up @@ -2118,8 +2101,6 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
};
}

let llvm_version = crate::llvm_util::get_version();

/// Converts a vector mask, where each element has a bit width equal to the data elements it is used with,
/// down to an i1 based mask that can be used by llvm intrinsics.
///
Expand Down Expand Up @@ -2666,22 +2647,16 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
// Type of the vector of elements:
let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len);

let args: &[&'ll Value] = if llvm_version < (22, 0, 0) {
let alignment = bx.const_i32(alignment as i32);
&[args[1].immediate(), alignment, mask, args[0].immediate()]
} else {
&[args[1].immediate(), mask, args[0].immediate()]
};

let call =
bx.call_intrinsic("llvm.masked.gather", &[llvm_elem_vec_ty, llvm_pointer_vec_ty], args);
if llvm_version >= (22, 0, 0) {
crate::attributes::apply_to_callsite(
call,
crate::llvm::AttributePlace::Argument(0),
&[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)],
)
}
let call = bx.call_intrinsic(
"llvm.masked.gather",
&[llvm_elem_vec_ty, llvm_pointer_vec_ty],
&[args[1].immediate(), mask, args[0].immediate()],
);
crate::attributes::apply_to_callsite(
call,
crate::llvm::AttributePlace::Argument(0),
&[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)],
);
return Ok(call);
}

Expand Down Expand Up @@ -2772,22 +2747,16 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
// Type of the vector of elements:
let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len);

let args: &[&'ll Value] = if llvm_version < (22, 0, 0) {
let alignment = bx.const_i32(alignment as i32);

&[args[1].immediate(), alignment, mask, args[2].immediate()]
} else {
&[args[1].immediate(), mask, args[2].immediate()]
};

let call = bx.call_intrinsic("llvm.masked.load", &[llvm_elem_vec_ty, llvm_pointer], args);
if llvm_version >= (22, 0, 0) {
crate::attributes::apply_to_callsite(
call,
crate::llvm::AttributePlace::Argument(0),
&[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)],
)
}
let call = bx.call_intrinsic(
"llvm.masked.load",
&[llvm_elem_vec_ty, llvm_pointer],
&[args[1].immediate(), mask, args[2].immediate()],
);
crate::attributes::apply_to_callsite(
call,
crate::llvm::AttributePlace::Argument(0),
&[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)],
);
return Ok(call);
}

Expand Down Expand Up @@ -2859,21 +2828,16 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
// Type of the vector of elements:
let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len);

let args: &[&'ll Value] = if llvm_version < (22, 0, 0) {
let alignment = bx.const_i32(alignment as i32);
&[args[2].immediate(), args[1].immediate(), alignment, mask]
} else {
&[args[2].immediate(), args[1].immediate(), mask]
};

let call = bx.call_intrinsic("llvm.masked.store", &[llvm_elem_vec_ty, llvm_pointer], args);
if llvm_version >= (22, 0, 0) {
crate::attributes::apply_to_callsite(
call,
crate::llvm::AttributePlace::Argument(1),
&[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)],
)
}
let call = bx.call_intrinsic(
"llvm.masked.store",
&[llvm_elem_vec_ty, llvm_pointer],
&[args[2].immediate(), args[1].immediate(), mask],
);
crate::attributes::apply_to_callsite(
call,
crate::llvm::AttributePlace::Argument(1),
&[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)],
);
return Ok(call);
}

Expand Down Expand Up @@ -2949,24 +2913,17 @@ fn generic_simd_intrinsic<'ll, 'tcx>(

// Type of the vector of elements:
let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len);
let args: &[&'ll Value] = if llvm_version < (22, 0, 0) {
let alignment = bx.const_i32(alignment as i32);
&[args[0].immediate(), args[1].immediate(), alignment, mask]
} else {
&[args[0].immediate(), args[1].immediate(), mask]
};

let call = bx.call_intrinsic(
"llvm.masked.scatter",
&[llvm_elem_vec_ty, llvm_pointer_vec_ty],
args,
&[args[0].immediate(), args[1].immediate(), mask],
);
crate::attributes::apply_to_callsite(
call,
crate::llvm::AttributePlace::Argument(1),
&[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)],
);
if llvm_version >= (22, 0, 0) {
crate::attributes::apply_to_callsite(
call,
crate::llvm::AttributePlace::Argument(1),
&[crate::llvm::CreateAlignmentAttr(bx.llcx, alignment)],
)
}
return Ok(call);
}

Expand Down Expand Up @@ -3448,17 +3405,11 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
&[vec_ty],
&[args[0].immediate(), args[1].immediate(), args[2].immediate()],
)),
sym::simd_carryless_mul => {
if crate::llvm_util::get_version() >= (22, 0, 0) {
Ok(bx.call_intrinsic(
llvm_intrinsic,
&[vec_ty],
&[args[0].immediate(), args[1].immediate()],
))
} else {
span_bug!(span, "`simd_carryless_mul` needs LLVM 22 or higher");
}
}
sym::simd_carryless_mul => Ok(bx.call_intrinsic(
llvm_intrinsic,
&[vec_ty],
&[args[0].immediate(), args[1].immediate()],
)),
_ => unreachable!(),
};
}
Expand Down
Loading
Loading