rustc_mir_transform/
lower_intrinsics.rs

1//! Lowers intrinsic calls
2
3use rustc_middle::mir::*;
4use rustc_middle::ty::{self, TyCtxt};
5use rustc_middle::{bug, span_bug};
6use rustc_span::sym;
7
8use crate::take_array;
9
10pub(super) struct LowerIntrinsics;
11
12impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics {
13    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
14        let local_decls = &body.local_decls;
15        for block in body.basic_blocks.as_mut() {
16            let terminator = block.terminator.as_mut().unwrap();
17            if let TerminatorKind::Call { func, args, destination, target, .. } =
18                &mut terminator.kind
19                && let ty::FnDef(def_id, generic_args) = *func.ty(local_decls, tcx).kind()
20                && let Some(intrinsic) = tcx.intrinsic(def_id)
21            {
22                match intrinsic.name {
23                    sym::unreachable => {
24                        terminator.kind = TerminatorKind::Unreachable;
25                    }
26                    sym::ub_checks | sym::overflow_checks | sym::contract_checks => {
27                        let op = match intrinsic.name {
28                            sym::ub_checks => RuntimeChecks::UbChecks,
29                            sym::contract_checks => RuntimeChecks::ContractChecks,
30                            sym::overflow_checks => RuntimeChecks::OverflowChecks,
31                            _ => unreachable!(),
32                        };
33                        let target = target.unwrap();
34                        block.statements.push(Statement::new(
35                            terminator.source_info,
36                            StatementKind::Assign(Box::new((
37                                *destination,
38                                Rvalue::Use(Operand::RuntimeChecks(op), WithRetag::Yes),
39                            ))),
40                        ));
41                        terminator.kind = TerminatorKind::Goto { target };
42                    }
43                    sym::forget => {
44                        let target = target.unwrap();
45                        block.statements.push(Statement::new(
46                            terminator.source_info,
47                            StatementKind::Assign(Box::new((
48                                *destination,
49                                Rvalue::Use(
50                                    Operand::Constant(Box::new(ConstOperand {
51                                        span: terminator.source_info.span,
52                                        user_ty: None,
53                                        const_: Const::zero_sized(tcx.types.unit),
54                                    })),
55                                    WithRetag::Yes,
56                                ),
57                            ))),
58                        ));
59                        terminator.kind = TerminatorKind::Goto { target };
60                    }
61                    sym::copy_nonoverlapping => {
62                        let target = target.unwrap();
63                        let Ok([src, dst, count]) = take_array(args) else {
64                            bug!("Wrong arguments for copy_non_overlapping intrinsic");
65                        };
66                        block.statements.push(Statement::new(
67                            terminator.source_info,
68                            StatementKind::Intrinsic(Box::new(
69                                NonDivergingIntrinsic::CopyNonOverlapping(
70                                    rustc_middle::mir::CopyNonOverlapping {
71                                        src: src.node,
72                                        dst: dst.node,
73                                        count: count.node,
74                                    },
75                                ),
76                            )),
77                        ));
78                        terminator.kind = TerminatorKind::Goto { target };
79                    }
80                    sym::assume => {
81                        let target = target.unwrap();
82                        let Ok([arg]) = take_array(args) else {
83                            bug!("Wrong arguments for assume intrinsic");
84                        };
85                        block.statements.push(Statement::new(
86                            terminator.source_info,
87                            StatementKind::Intrinsic(Box::new(NonDivergingIntrinsic::Assume(
88                                arg.node,
89                            ))),
90                        ));
91                        terminator.kind = TerminatorKind::Goto { target };
92                    }
93                    sym::wrapping_add
94                    | sym::wrapping_sub
95                    | sym::wrapping_mul
96                    | sym::three_way_compare
97                    | sym::unchecked_add
98                    | sym::unchecked_sub
99                    | sym::unchecked_mul
100                    | sym::unchecked_div
101                    | sym::unchecked_rem
102                    | sym::unchecked_shl
103                    | sym::unchecked_shr => {
104                        let target = target.unwrap();
105                        let Ok([lhs, rhs]) = take_array(args) else {
106                            bug!("Wrong arguments for {} intrinsic", intrinsic.name);
107                        };
108                        let bin_op = match intrinsic.name {
109                            sym::wrapping_add => BinOp::Add,
110                            sym::wrapping_sub => BinOp::Sub,
111                            sym::wrapping_mul => BinOp::Mul,
112                            sym::three_way_compare => BinOp::Cmp,
113                            sym::unchecked_add => BinOp::AddUnchecked,
114                            sym::unchecked_sub => BinOp::SubUnchecked,
115                            sym::unchecked_mul => BinOp::MulUnchecked,
116                            sym::unchecked_div => BinOp::Div,
117                            sym::unchecked_rem => BinOp::Rem,
118                            sym::unchecked_shl => BinOp::ShlUnchecked,
119                            sym::unchecked_shr => BinOp::ShrUnchecked,
120                            _ => bug!("unexpected intrinsic"),
121                        };
122                        block.statements.push(Statement::new(
123                            terminator.source_info,
124                            StatementKind::Assign(Box::new((
125                                *destination,
126                                Rvalue::BinaryOp(bin_op, Box::new((lhs.node, rhs.node))),
127                            ))),
128                        ));
129                        terminator.kind = TerminatorKind::Goto { target };
130                    }
131                    sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => {
132                        let target = target.unwrap();
133                        let Ok([lhs, rhs]) = take_array(args) else {
134                            bug!("Wrong arguments for {} intrinsic", intrinsic.name);
135                        };
136                        let bin_op = match intrinsic.name {
137                            sym::add_with_overflow => BinOp::AddWithOverflow,
138                            sym::sub_with_overflow => BinOp::SubWithOverflow,
139                            sym::mul_with_overflow => BinOp::MulWithOverflow,
140                            _ => bug!("unexpected intrinsic"),
141                        };
142                        block.statements.push(Statement::new(
143                            terminator.source_info,
144                            StatementKind::Assign(Box::new((
145                                *destination,
146                                Rvalue::BinaryOp(bin_op, Box::new((lhs.node, rhs.node))),
147                            ))),
148                        ));
149                        terminator.kind = TerminatorKind::Goto { target };
150                    }
151                    sym::read_via_copy => {
152                        let Ok([arg]) = take_array(args) else {
153                            span_bug!(terminator.source_info.span, "Wrong number of arguments");
154                        };
155                        let derefed_place = if let Some(place) = arg.node.place()
156                            && let Some(local) = place.as_local()
157                        {
158                            tcx.mk_place_deref(local.into())
159                        } else {
160                            span_bug!(
161                                terminator.source_info.span,
162                                "Only passing a local is supported"
163                            );
164                        };
165                        // Add new statement at the end of the block that does the read, and patch
166                        // up the terminator.
167                        block.statements.push(Statement::new(
168                            terminator.source_info,
169                            StatementKind::Assign(Box::new((
170                                *destination,
171                                Rvalue::Use(Operand::Copy(derefed_place), WithRetag::Yes),
172                            ))),
173                        ));
174                        terminator.kind = match *target {
175                            None => {
176                                // No target means this read something uninhabited,
177                                // so it must be unreachable.
178                                TerminatorKind::Unreachable
179                            }
180                            Some(target) => TerminatorKind::Goto { target },
181                        }
182                    }
183                    // `write_via_move` is already lowered during MIR building.
184                    sym::discriminant_value => {
185                        let target = target.unwrap();
186                        let Ok([arg]) = take_array(args) else {
187                            span_bug!(
188                                terminator.source_info.span,
189                                "Wrong arguments for discriminant_value intrinsic"
190                            );
191                        };
192                        let arg = arg.node.place().unwrap();
193                        let arg = tcx.mk_place_deref(arg);
194                        block.statements.push(Statement::new(
195                            terminator.source_info,
196                            StatementKind::Assign(Box::new((
197                                *destination,
198                                Rvalue::Discriminant(arg),
199                            ))),
200                        ));
201                        terminator.kind = TerminatorKind::Goto { target };
202                    }
203                    sym::offset => {
204                        let target = target.unwrap();
205                        let Ok([ptr, delta]) = take_array(args) else {
206                            span_bug!(
207                                terminator.source_info.span,
208                                "Wrong number of arguments for offset intrinsic",
209                            );
210                        };
211                        block.statements.push(Statement::new(
212                            terminator.source_info,
213                            StatementKind::Assign(Box::new((
214                                *destination,
215                                Rvalue::BinaryOp(BinOp::Offset, Box::new((ptr.node, delta.node))),
216                            ))),
217                        ));
218                        terminator.kind = TerminatorKind::Goto { target };
219                    }
220                    sym::slice_get_unchecked => {
221                        let target = target.unwrap();
222                        let Ok([ptrish, index]) = take_array(args) else {
223                            span_bug!(
224                                terminator.source_info.span,
225                                "Wrong number of arguments for {intrinsic:?}",
226                            );
227                        };
228
229                        let place = ptrish.node.place().unwrap();
230                        assert!(!place.is_indirect());
231                        let updated_place = place.project_deeper(
232                            &[
233                                ProjectionElem::Deref,
234                                ProjectionElem::Index(
235                                    index.node.place().unwrap().as_local().unwrap(),
236                                ),
237                            ],
238                            tcx,
239                        );
240
241                        let ret_ty = generic_args.type_at(0);
242                        let rvalue = match *ret_ty.kind() {
243                            ty::RawPtr(_, Mutability::Not) => {
244                                Rvalue::RawPtr(RawPtrKind::Const, updated_place)
245                            }
246                            ty::RawPtr(_, Mutability::Mut) => {
247                                Rvalue::RawPtr(RawPtrKind::Mut, updated_place)
248                            }
249                            ty::Ref(region, _, Mutability::Not) => {
250                                Rvalue::Ref(region, BorrowKind::Shared, updated_place)
251                            }
252                            ty::Ref(region, _, Mutability::Mut) => Rvalue::Ref(
253                                region,
254                                BorrowKind::Mut { kind: MutBorrowKind::Default },
255                                updated_place,
256                            ),
257                            _ => bug!("Unknown return type {ret_ty:?}"),
258                        };
259
260                        block.statements.push(Statement::new(
261                            terminator.source_info,
262                            StatementKind::Assign(Box::new((*destination, rvalue))),
263                        ));
264                        terminator.kind = TerminatorKind::Goto { target };
265                    }
266                    sym::transmute | sym::transmute_unchecked => {
267                        let dst_ty = destination.ty(local_decls, tcx).ty;
268                        let Ok([arg]) = take_array(args) else {
269                            span_bug!(
270                                terminator.source_info.span,
271                                "Wrong number of arguments for transmute intrinsic",
272                            );
273                        };
274
275                        // Always emit the cast, even if we transmute to an uninhabited type,
276                        // because that lets CTFE and codegen generate better error messages
277                        // when such a transmute actually ends up reachable.
278                        block.statements.push(Statement::new(
279                            terminator.source_info,
280                            StatementKind::Assign(Box::new((
281                                *destination,
282                                Rvalue::Cast(CastKind::Transmute, arg.node, dst_ty),
283                            ))),
284                        ));
285                        if let Some(target) = *target {
286                            terminator.kind = TerminatorKind::Goto { target };
287                        } else {
288                            terminator.kind = TerminatorKind::Unreachable;
289                        }
290                    }
291                    sym::aggregate_raw_ptr => {
292                        let Ok([data, meta]) = take_array(args) else {
293                            span_bug!(
294                                terminator.source_info.span,
295                                "Wrong number of arguments for aggregate_raw_ptr intrinsic",
296                            );
297                        };
298                        let target = target.unwrap();
299                        let pointer_ty = generic_args.type_at(0);
300                        let kind = if let ty::RawPtr(pointee_ty, mutability) = pointer_ty.kind() {
301                            AggregateKind::RawPtr(*pointee_ty, *mutability)
302                        } else {
303                            span_bug!(
304                                terminator.source_info.span,
305                                "Return type of aggregate_raw_ptr intrinsic must be a raw pointer",
306                            );
307                        };
308                        let fields = [data.node, meta.node];
309                        block.statements.push(Statement::new(
310                            terminator.source_info,
311                            StatementKind::Assign(Box::new((
312                                *destination,
313                                Rvalue::Aggregate(Box::new(kind), fields.into()),
314                            ))),
315                        ));
316                        terminator.kind = TerminatorKind::Goto { target };
317                    }
318                    sym::ptr_metadata => {
319                        let Ok([ptr]) = take_array(args) else {
320                            span_bug!(
321                                terminator.source_info.span,
322                                "Wrong number of arguments for ptr_metadata intrinsic",
323                            );
324                        };
325                        let target = target.unwrap();
326                        block.statements.push(Statement::new(
327                            terminator.source_info,
328                            StatementKind::Assign(Box::new((
329                                *destination,
330                                Rvalue::UnaryOp(UnOp::PtrMetadata, ptr.node),
331                            ))),
332                        ));
333                        terminator.kind = TerminatorKind::Goto { target };
334                    }
335                    _ => {}
336                }
337            }
338        }
339    }
340
341    fn is_required(&self) -> bool {
342        true
343    }
344}
rustc_mir_transform/lower_intrinsics.rs

rustc_mir_transform/
lower_intrinsics.rs
