Skip to content

Commit 7eb36f3

Browse files
GeorgeWortGeorgeWort
committed
Allow constants using an Abi::Vector layout to be passed to the backend
This allows constant vectors using a repr(simd) type to be propagated through to the backend. This fixes a few issues with LLVM simd intrinsics where a constant vector is expected but cannot be produced by rust at opt-level=0.
1 parent fa14810 commit 7eb36f3

File tree

7 files changed

+138
-50
lines changed

7 files changed

+138
-50
lines changed

‎compiler/rustc_codegen_gcc/src/type_.rs

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -10,22 +10,6 @@ use crate::context::CodegenCx;
1010
use crate::type_of::LayoutGccExt;
1111

1212
impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
13-
pub fn type_ix(&self, num_bits: u64) -> Type<'gcc> {
14-
// gcc only supports 1, 2, 4 or 8-byte integers.
15-
// FIXME(antoyo): this is misleading to use the next power of two as rustc_codegen_ssa
16-
// sometimes use 96-bit numbers and the following code will give an integer of a different
17-
// size.
18-
let bytes = (num_bits / 8).next_power_of_two() as i32;
19-
match bytes {
20-
1 => self.i8_type,
21-
2 => self.i16_type,
22-
4 => self.i32_type,
23-
8 => self.i64_type,
24-
16 => self.i128_type,
25-
_ => panic!("unexpected num_bits: {}", num_bits),
26-
}
27-
}
28-
2913
pub fn type_void(&self) -> Type<'gcc> {
3014
self.context.new_type::<()>()
3115
}
@@ -90,6 +74,22 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
9074
}
9175

9276
impl<'gcc, 'tcx> BaseTypeMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
77+
fn type_ix(&self, num_bits: u64) -> Type<'gcc> {
78+
// gcc only supports 1, 2, 4 or 8-byte integers.
79+
// FIXME(antoyo): this is misleading to use the next power of two as rustc_codegen_ssa
80+
// sometimes use 96-bit numbers and the following code will give an integer of a different
81+
// size.
82+
let bytes = (num_bits / 8).next_power_of_two() as i32;
83+
match bytes {
84+
1 => self.i8_type,
85+
2 => self.i16_type,
86+
4 => self.i32_type,
87+
8 => self.i64_type,
88+
16 => self.i128_type,
89+
_ => panic!("unexpected num_bits: {}", num_bits),
90+
}
91+
}
92+
9393
fn type_i1(&self) -> Type<'gcc> {
9494
self.bool_type
9595
}

‎compiler/rustc_codegen_llvm/src/type_.rs

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -60,11 +60,6 @@ impl<'ll> CodegenCx<'ll, '_> {
6060
unsafe { llvm::LLVMMetadataTypeInContext(self.llcx) }
6161
}
6262

63-
///x Creates an integer type with the given number of bits, e.g., i24
64-
pub(crate) fn type_ix(&self, num_bits: u64) -> &'ll Type {
65-
unsafe { llvm::LLVMIntTypeInContext(self.llcx, num_bits as c_uint) }
66-
}
67-
6863
pub(crate) fn type_vector(&self, ty: &'ll Type, len: u64) -> &'ll Type {
6964
unsafe { llvm::LLVMVectorType(ty, len as c_uint) }
7065
}
@@ -128,6 +123,11 @@ impl<'ll> CodegenCx<'ll, '_> {
128123
}
129124

130125
impl<'ll, 'tcx> BaseTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> {
126+
/// Creates an integer type with the given number of bits, e.g., i24
127+
fn type_ix(&self, num_bits: u64) -> &'ll Type {
128+
unsafe { llvm::LLVMIntTypeInContext(self.llcx, num_bits as c_uint) }
129+
}
130+
131131
fn type_i1(&self) -> &'ll Type {
132132
unsafe { llvm::LLVMInt1TypeInContext(self.llcx) }
133133
}

‎compiler/rustc_codegen_ssa/src/mir/operand.rs

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -92,13 +92,29 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
9292
let layout = bx.layout_of(ty);
9393

9494
let val = match val {
95-
ConstValue::Scalar(x) => {
96-
let Abi::Scalar(scalar) = layout.abi else {
97-
bug!("from_const: invalid ByVal layout: {:#?}", layout);
98-
};
99-
let llval = bx.scalar_to_backend(x, scalar, bx.immediate_backend_type(layout));
100-
OperandValue::Immediate(llval)
101-
}
95+
ConstValue::Scalar(x) => match layout.abi {
96+
Abi::Scalar(scalar) => {
97+
let llval = bx.scalar_to_backend(x, scalar, bx.immediate_backend_type(layout));
98+
OperandValue::Immediate(llval)
99+
}
100+
Abi::Vector {
101+
element: abi::Scalar::Initialized { value: abi::Primitive::Int(..), .. },
102+
..
103+
} if layout.size.bytes() <= 16 => {
104+
if let Scalar::Int(_) = x {
105+
let llval = bx.const_uint_big(
106+
bx.type_ix(layout.size.bits()),
107+
x.to_bits(layout.size).unwrap(),
108+
);
109+
OperandValue::Immediate(
110+
bx.const_bitcast(llval, bx.immediate_backend_type(layout)),
111+
)
112+
} else {
113+
bug!("Only Scalar::Int constant vectors are supported")
114+
}
115+
}
116+
_ => bug!("from_const: invalid ByVal layout: {:#?}", layout),
117+
},
102118
ConstValue::ZeroSized => return OperandRef::zero_sized(layout),
103119
ConstValue::Slice { data, meta } => {
104120
let Abi::ScalarPair(a_scalar, _) = layout.abi else {

‎compiler/rustc_codegen_ssa/src/traits/type_.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ use rustc_target::abi::{AddressSpace, Integer};
1111
// This depends on `Backend` and not `BackendTypes`, because consumers will probably want to use
1212
// `LayoutOf` or `HasTyCtxt`. This way, they don't have to add a constraint on it themselves.
1313
pub trait BaseTypeMethods<'tcx>: Backend<'tcx> {
14+
fn type_ix(&self, num_bits: u64) -> Self::Type;
1415
fn type_i1(&self) -> Self::Type;
1516
fn type_i8(&self) -> Self::Type;
1617
fn type_i16(&self) -> Self::Type;

‎compiler/rustc_const_eval/src/const_eval/eval_queries.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,10 @@ pub(super) fn op_to_const<'tcx>(
136136
// they can be stored as `ConstValue::Indirect`), but that's not relevant since we barely
137137
// ever have to do this. (`try_get_slice_bytes_for_diagnostics` exists to provide this
138138
// functionality.)
139+
Abi::Vector {
140+
element: abi::Scalar::Initialized { value: abi::Primitive::Int(..), .. },
141+
..
142+
} => op.layout.size.bytes() <= 16,
139143
_ => false,
140144
};
141145
let immediate = if force_as_immediate {

‎compiler/rustc_const_eval/src/interpret/operand.rs

Lines changed: 61 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,12 @@ use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
1010
use rustc_middle::ty::print::{FmtPrinter, PrettyPrinter};
1111
use rustc_middle::ty::{ConstInt, Ty, TyCtxt};
1212
use rustc_middle::{mir, ty};
13-
use rustc_target::abi::{self, Abi, HasDataLayout, Size};
13+
use rustc_target::abi::{self, Abi, HasDataLayout, Primitive, Size};
1414

1515
use super::{
1616
alloc_range, from_known_layout, mir_assign_valid_types, AllocId, Frame, InterpCx, InterpResult,
17-
MPlaceTy, Machine, MemPlace, MemPlaceMeta, OffsetMode, PlaceTy, Pointer, Projectable,
18-
Provenance, Scalar,
17+
MPlaceTy, Machine, MemPlace, MemPlaceMeta, MemoryKind, OffsetMode, PlaceTy, Pointer,
18+
Projectable, Provenance, Scalar,
1919
};
2020

2121
/// An `Immediate` represents a single immediate self-contained Rust value.
@@ -261,12 +261,29 @@ impl<'tcx, Prov: Provenance> ImmTy<'tcx, Prov> {
261261
{
262262
Immediate::Uninit
263263
}
264+
(
265+
Immediate::Scalar(scalar),
266+
Abi::Vector {
267+
element: abi::Scalar::Initialized { value: s @ Primitive::Int(..), .. },
268+
..
269+
},
270+
) => {
271+
let size = s.size(cx);
272+
assert_eq!(size, layout.size);
273+
assert!(self.layout.size.bytes() <= 16);
274+
let vector_bits: u128 = scalar.to_bits(self.layout.size).unwrap();
275+
Immediate::Scalar(Scalar::Int(
276+
ty::ScalarInt::try_from_uint(size.truncate(vector_bits >> offset.bits()), size)
277+
.unwrap(),
278+
))
279+
}
264280
// the field covers the entire type
265281
_ if layout.size == self.layout.size => {
266282
assert_eq!(offset.bytes(), 0);
267283
assert!(
268284
match (self.layout.abi, layout.abi) {
269285
(Abi::Scalar(..), Abi::Scalar(..)) => true,
286+
(Abi::Aggregate { sized: true }, Abi::Scalar(..)) => true,
270287
(Abi::ScalarPair(..), Abi::ScalarPair(..)) => true,
271288
_ => false,
272289
},
@@ -500,6 +517,21 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
500517
)?;
501518
Some(ImmTy::from_immediate(Immediate::ScalarPair(a_val, b_val), mplace.layout))
502519
}
520+
Abi::Vector {
521+
element: abi::Scalar::Initialized { value: s @ abi::Primitive::Int(..), .. },
522+
..
523+
} => {
524+
let size = s.size(self);
525+
let stride = size.align_to(s.align(self).abi);
526+
if mplace.layout.size.bytes() <= 16 {
527+
assert!(stride.bytes() > 0);
528+
let vector_scalar =
529+
alloc.read_scalar(alloc_range(Size::ZERO, mplace.layout.size), false)?;
530+
Some(ImmTy { imm: Immediate::Scalar(vector_scalar), layout: mplace.layout })
531+
} else {
532+
None
533+
}
534+
}
503535
_ => {
504536
// Neither a scalar nor scalar pair.
505537
None
@@ -539,12 +571,23 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
539571
&self,
540572
op: &impl Readable<'tcx, M::Provenance>,
541573
) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> {
542-
if !matches!(
543-
op.layout().abi,
574+
let can_primitive_read = match op.layout().abi {
544575
Abi::Scalar(abi::Scalar::Initialized { .. })
545-
| Abi::ScalarPair(abi::Scalar::Initialized { .. }, abi::Scalar::Initialized { .. })
546-
) {
547-
span_bug!(self.cur_span(), "primitive read not possible for type: {}", op.layout().ty);
576+
| Abi::ScalarPair(abi::Scalar::Initialized { .. }, abi::Scalar::Initialized { .. }) => {
577+
true
578+
}
579+
Abi::Vector {
580+
element: abi::Scalar::Initialized { value: abi::Primitive::Int(..), .. },
581+
..
582+
} => op.layout().size.bytes() <= 16,
583+
_ => false,
584+
};
585+
if !can_primitive_read {
586+
span_bug!(
587+
self.cur_span(),
588+
"primitive read not possible for type: {:?}",
589+
op.layout().ty
590+
);
548591
}
549592
let imm = self.read_immediate_raw(op)?.right().unwrap();
550593
if matches!(*imm, Immediate::Uninit) {
@@ -599,7 +642,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
599642
///
600643
/// Can (but does not always) trigger UB if `op` is uninitialized.
601644
pub fn operand_to_simd(
602-
&self,
645+
&mut self,
603646
op: &OpTy<'tcx, M::Provenance>,
604647
) -> InterpResult<'tcx, (MPlaceTy<'tcx, M::Provenance>, u64)> {
605648
// Basically we just transmute this place into an array following simd_size_and_type.
@@ -612,7 +655,15 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
612655
throw_ub!(InvalidUninitBytes(None))
613656
}
614657
Immediate::Scalar(..) | Immediate::ScalarPair(..) => {
615-
bug!("arrays/slices can never have Scalar/ScalarPair layout")
658+
if let Abi::Vector {
659+
element: abi::Scalar::Initialized { value: abi::Primitive::Int(..), .. }, ..
660+
} = op.layout.abi && op.layout.size.bytes() <= 16 {
661+
let mplace = self.allocate(op.layout, MemoryKind::Stack)?;
662+
self.write_immediate(imm.imm, &mplace).unwrap();
663+
self.mplace_to_simd(&mplace)
664+
} else {
665+
bug!("arrays/slices can never have Scalar/ScalarPair layout")
666+
}
616667
}
617668
},
618669
}

‎compiler/rustc_const_eval/src/interpret/place.rs

Lines changed: 28 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@ use rustc_middle::mir;
1212
use rustc_middle::ty;
1313
use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
1414
use rustc_middle::ty::Ty;
15-
use rustc_target::abi::{Abi, Align, FieldIdx, HasDataLayout, Size, FIRST_VARIANT};
15+
use rustc_target::abi;
16+
use rustc_target::abi::{Abi, Align, FieldIdx, HasDataLayout, Primitive, Size, FIRST_VARIANT};
1617

1718
use super::{
1819
alloc_range, mir_assign_valid_types, AllocId, AllocRef, AllocRefMut, CheckAlignMsg, ImmTy,
@@ -696,17 +697,32 @@ where
696697
};
697698

698699
match value {
699-
Immediate::Scalar(scalar) => {
700-
let Abi::Scalar(s) = layout.abi else {
701-
span_bug!(
702-
self.cur_span(),
703-
"write_immediate_to_mplace: invalid Scalar layout: {layout:#?}",
704-
)
705-
};
706-
let size = s.size(&tcx);
707-
assert_eq!(size, layout.size, "abi::Scalar size does not match layout size");
708-
alloc.write_scalar(alloc_range(Size::ZERO, size), scalar)
709-
}
700+
Immediate::Scalar(scalar) => match layout.abi {
701+
Abi::Scalar(s) => {
702+
let size = s.size(&tcx);
703+
assert_eq!(size, layout.size, "abi::Scalar size does not match layout size");
704+
alloc.write_scalar(alloc_range(Size::ZERO, size), scalar)
705+
}
706+
Abi::Vector {
707+
element: abi::Scalar::Initialized { value: s @ Primitive::Int(..), .. },
708+
..
709+
} => {
710+
let size = s.size(&tcx);
711+
let stride = size.align_to(s.align(&tcx).abi);
712+
assert!(stride.bytes() > 0);
713+
if layout.size.bytes() > 16 {
714+
span_bug!(
715+
self.cur_span(),
716+
"write_immediate_to_mplace: invalid Vector layout: {layout:#?}",
717+
)
718+
}
719+
alloc.write_scalar(alloc_range(Size::ZERO, layout.size), scalar)
720+
}
721+
_ => span_bug!(
722+
self.cur_span(),
723+
"write_immediate_to_mplace: invalid Scalar layout: {layout:#?}",
724+
),
725+
},
710726
Immediate::ScalarPair(a_val, b_val) => {
711727
let Abi::ScalarPair(a, b) = layout.abi else {
712728
span_bug!(

0 commit comments

Comments
 (0)