Skip to content

Commit 4794ea1

Browse files
RalfJungRalfJung
committed
atomic_load intrinsic: use const generic parameter for ordering
1 parent 6f69710 commit 4794ea1

File tree

15 files changed

+198
-74
lines changed

15 files changed

+198
-74
lines changed

‎compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -870,11 +870,12 @@ fn codegen_regular_intrinsic_call<'tcx>(
870870
// FIXME use a compiler fence once Cranelift supports it
871871
fx.bcx.ins().fence();
872872
}
873-
_ if intrinsic.as_str().starts_with("atomic_load") => {
873+
sym::atomic_load => {
874874
intrinsic_args!(fx, args => (ptr); intrinsic);
875875
let ptr = ptr.load_scalar(fx);
876876

877877
let ty = generic_args.type_at(0);
878+
let _ord = generic_args.const_at(1).to_value(); // FIXME: forward this to cranelift once they support that
878879
match ty.kind() {
879880
ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => {
880881
// FIXME implement 128bit atomics

‎compiler/rustc_codegen_ssa/src/mir/intrinsic.rs‎

Lines changed: 59 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,17 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
9999

100100
let llret_ty = bx.backend_type(bx.layout_of(ret_ty));
101101

102+
let ret_llval = |bx: &mut Bx, llval| {
103+
if result.layout.ty.is_bool() {
104+
OperandRef::from_immediate_or_packed_pair(bx, llval, result.layout)
105+
.val
106+
.store(bx, result);
107+
} else if !result.layout.ty.is_unit() {
108+
bx.store_to_place(llval, result.val);
109+
}
110+
Ok(())
111+
};
112+
102113
let llval = match name {
103114
sym::abort => {
104115
bx.abort();
@@ -337,6 +348,53 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
337348
use crate::common::AtomicOrdering::*;
338349
use crate::common::{AtomicRmwBinOp, SynchronizationScope};
339350

351+
let invalid_monomorphization = |ty| {
352+
bx.tcx().dcx().emit_err(InvalidMonomorphization::BasicIntegerType {
353+
span,
354+
name,
355+
ty,
356+
});
357+
};
358+
359+
let parse_const_generic_ordering = |ord: ty::Value<'tcx>| {
360+
let discr = ord.valtree.unwrap_branch()[0].unwrap_leaf();
361+
let ord = discr.to_atomic_ordering();
362+
// We have to translate from the intrinsic ordering to the backend ordering.
363+
use rustc_middle::ty::AtomicOrdering;
364+
match ord {
365+
AtomicOrdering::Relaxed => Relaxed,
366+
AtomicOrdering::Release => Release,
367+
AtomicOrdering::Acquire => Acquire,
368+
AtomicOrdering::AcqRel => AcquireRelease,
369+
AtomicOrdering::SeqCst => SequentiallyConsistent,
370+
}
371+
};
372+
373+
// Some intrinsics have the ordering already converted to a const generic parameter, we handle those first.
374+
match name {
375+
sym::atomic_load => {
376+
let ty = fn_args.type_at(0);
377+
let ordering = fn_args.const_at(1).to_value();
378+
if !(int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr()) {
379+
invalid_monomorphization(ty);
380+
return Ok(());
381+
}
382+
let layout = bx.layout_of(ty);
383+
let source = args[0].immediate();
384+
let llval = bx.atomic_load(
385+
bx.backend_type(layout),
386+
source,
387+
parse_const_generic_ordering(ordering),
388+
layout.size,
389+
);
390+
391+
return ret_llval(bx, llval);
392+
}
393+
394+
// The rest falls back to below.
395+
_ => {}
396+
}
397+
340398
let Some((instruction, ordering)) = atomic.split_once('_') else {
341399
bx.sess().dcx().emit_fatal(errors::MissingMemoryOrdering);
342400
};
@@ -350,14 +408,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
350408
_ => bx.sess().dcx().emit_fatal(errors::UnknownAtomicOrdering),
351409
};
352410

353-
let invalid_monomorphization = |ty| {
354-
bx.tcx().dcx().emit_err(InvalidMonomorphization::BasicIntegerType {
355-
span,
356-
name,
357-
ty,
358-
});
359-
};
360-
361411
match instruction {
362412
"cxchg" | "cxchgweak" => {
363413
let Some((success, failure)) = ordering.split_once('_') else {
@@ -390,24 +440,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
390440
return Ok(());
391441
}
392442

393-
"load" => {
394-
let ty = fn_args.type_at(0);
395-
if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() {
396-
let layout = bx.layout_of(ty);
397-
let size = layout.size;
398-
let source = args[0].immediate();
399-
bx.atomic_load(
400-
bx.backend_type(layout),
401-
source,
402-
parse_ordering(bx, ordering),
403-
size,
404-
)
405-
} else {
406-
invalid_monomorphization(ty);
407-
return Ok(());
408-
}
409-
}
410-
411443
"store" => {
412444
let ty = fn_args.type_at(0);
413445
if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_raw_ptr() {
@@ -538,14 +570,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
538570
}
539571
};
540572

541-
if result.layout.ty.is_bool() {
542-
OperandRef::from_immediate_or_packed_pair(bx, llval, result.layout)
543-
.val
544-
.store(bx, result);
545-
} else if !result.layout.ty.is_unit() {
546-
bx.store_to_place(llval, result.val);
547-
}
548-
Ok(())
573+
ret_llval(bx, llval)
549574
}
550575
}
551576

‎compiler/rustc_hir_analysis/src/check/intrinsic.rs‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -204,24 +204,25 @@ pub(crate) fn check_intrinsic_type(
204204

205205
// Each atomic op has variants with different suffixes (`_seq_cst`, `_acquire`, etc.). Use
206206
// string ops to strip the suffixes, because the variants all get the same treatment here.
207-
let (n_tps, inputs, output) = match split[1] {
207+
let (n_tps, n_cts, inputs, output) = match split[1] {
208208
"cxchg" | "cxchgweak" => (
209209
1,
210+
0,
210211
vec![Ty::new_mut_ptr(tcx, param(0)), param(0), param(0)],
211212
Ty::new_tup(tcx, &[param(0), tcx.types.bool]),
212213
),
213-
"load" => (1, vec![Ty::new_imm_ptr(tcx, param(0))], param(0)),
214-
"store" => (1, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], tcx.types.unit),
214+
"load" => (1, 1, vec![Ty::new_imm_ptr(tcx, param(0))], param(0)),
215+
"store" => (1, 0, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], tcx.types.unit),
215216

216217
"xchg" | "xadd" | "xsub" | "and" | "nand" | "or" | "xor" | "max" | "min" | "umax"
217-
| "umin" => (1, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], param(0)),
218-
"fence" | "singlethreadfence" => (0, Vec::new(), tcx.types.unit),
218+
| "umin" => (1, 0, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], param(0)),
219+
"fence" | "singlethreadfence" => (0, 0, Vec::new(), tcx.types.unit),
219220
op => {
220221
tcx.dcx().emit_err(UnrecognizedAtomicOperation { span, op });
221222
return;
222223
}
223224
};
224-
(n_tps, 0, 0, inputs, output, hir::Safety::Unsafe)
225+
(n_tps, 0, n_cts, inputs, output, hir::Safety::Unsafe)
225226
} else if intrinsic_name == sym::contract_check_ensures {
226227
// contract_check_ensures::<Ret, C>(Ret, C) -> Ret
227228
// where C: for<'a> Fn(&'a Ret) -> bool,

‎compiler/rustc_middle/src/ty/consts/int.rs‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,19 @@ impl ConstInt {
2626
}
2727
}
2828

29+
/// An enum to represent the compiler-side view of `intrinsics::AtomicOrdering`.
30+
/// This lives here because there's a method in this file that needs it and it is entirely unclear
31+
/// where else to put this...
32+
#[derive(Debug)]
33+
pub enum AtomicOrdering {
34+
// These values must match `intrinsics::AtomicOrdering`!
35+
Relaxed = 0,
36+
Release = 1,
37+
Acquire = 2,
38+
AcqRel = 3,
39+
SeqCst = 4,
40+
}
41+
2942
impl std::fmt::Debug for ConstInt {
3043
fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
3144
let Self { int, signed, is_ptr_sized_integral } = *self;
@@ -318,6 +331,25 @@ impl ScalarInt {
318331
self.to_uint(tcx.data_layout.pointer_size).try_into().unwrap()
319332
}
320333

334+
#[inline]
335+
pub fn to_atomic_ordering(self) -> AtomicOrdering {
336+
use AtomicOrdering::*;
337+
let val = self.to_u32();
338+
if val == Relaxed as u32 {
339+
Relaxed
340+
} else if val == Release as u32 {
341+
Release
342+
} else if val == Acquire as u32 {
343+
Acquire
344+
} else if val == AcqRel as u32 {
345+
AcqRel
346+
} else if val == SeqCst as u32 {
347+
SeqCst
348+
} else {
349+
panic!("not a valid atomic ordering")
350+
}
351+
}
352+
321353
/// Converts the `ScalarInt` to `bool`.
322354
/// Panics if the `size` of the `ScalarInt` is not equal to 1 byte.
323355
/// Errors if it is not a valid `bool`.
@@ -488,7 +520,7 @@ from_scalar_int_for_x_signed!(i8, i16, i32, i64, i128);
488520
impl From<std::cmp::Ordering> for ScalarInt {
489521
#[inline]
490522
fn from(c: std::cmp::Ordering) -> Self {
491-
// Here we rely on `Ordering` having the same values in host and target!
523+
// Here we rely on `cmp::Ordering` having the same values in host and target!
492524
ScalarInt::from(c as i8)
493525
}
494526
}

‎compiler/rustc_middle/src/ty/mod.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -74,8 +74,8 @@ pub use self::closure::{
7474
place_to_string_for_capture,
7575
};
7676
pub use self::consts::{
77-
AnonConstKind, Const, ConstInt, ConstKind, Expr, ExprKind, ScalarInt, UnevaluatedConst,
78-
ValTree, ValTreeKind, Value,
77+
AnonConstKind, AtomicOrdering, Const, ConstInt, ConstKind, Expr, ExprKind, ScalarInt,
78+
UnevaluatedConst, ValTree, ValTreeKind, Value,
7979
};
8080
pub use self::context::{
8181
CtxtInterners, CurrentGcx, DeducedParamAttrs, Feed, FreeRegionInfo, GlobalCtxt, Lift, TyCtxt,

‎compiler/rustc_span/src/symbol.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -515,6 +515,7 @@ symbols! {
515515
async_iterator_poll_next,
516516
async_trait_bounds,
517517
atomic,
518+
atomic_load,
518519
atomic_mod,
519520
atomics,
520521
att_syntax,

‎library/core/src/intrinsics/mod.rs‎

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030
//!
3131
//! The atomic intrinsics provide common atomic operations on machine
3232
//! words, with multiple possible memory orderings. See the
33-
//! [atomic types][crate::sync::atomic] docs for details.
33+
//! [atomic types][atomic] docs for details.
3434
//!
3535
//! # Unwinding
3636
//!
@@ -50,7 +50,7 @@
5050
)]
5151
#![allow(missing_docs)]
5252

53-
use crate::marker::{DiscriminantKind, Tuple};
53+
use crate::marker::{ConstParamTy, DiscriminantKind, Tuple};
5454
use crate::ptr;
5555

5656
pub mod fallback;
@@ -62,6 +62,20 @@ pub mod simd;
6262
#[cfg(all(target_has_atomic = "8", target_has_atomic = "32", target_has_atomic = "ptr"))]
6363
use crate::sync::atomic::{self, AtomicBool, AtomicI32, AtomicIsize, AtomicU32, Ordering};
6464

65+
/// A type for atomic ordering parameters for intrinsics. This is a separate type from
66+
/// `atomic::Ordering` so that we can make it `ConstParamTy` and fix the values used here without a
67+
/// risk of leaking that to stable code.
68+
#[derive(Debug, ConstParamTy, PartialEq, Eq)]
69+
pub enum AtomicOrdering {
70+
// These values must match the compiler's `AtomicOrdering` defined in
71+
// `rustc_middle/src/ty/consts/int.rs`!
72+
Relaxed = 0,
73+
Release = 1,
74+
Acquire = 2,
75+
AcqRel = 3,
76+
SeqCst = 4,
77+
}
78+
6579
// N.B., these intrinsics take raw pointers because they mutate aliased
6680
// memory, which is not valid for either `&` or `&mut`.
6781

@@ -391,6 +405,15 @@ pub unsafe fn atomic_cxchgweak_seqcst_acquire<T: Copy>(dst: *mut T, old: T, src:
391405
#[rustc_nounwind]
392406
pub unsafe fn atomic_cxchgweak_seqcst_seqcst<T: Copy>(dst: *mut T, old: T, src: T) -> (T, bool);
393407

408+
/// Loads the current value of the pointer.
409+
/// `T` must be an integer or pointer type.
410+
///
411+
/// The stabilized version of this intrinsic is available on the
412+
/// [`atomic`] types via the `load` method. For example, [`AtomicBool::load`].
413+
#[rustc_intrinsic]
414+
#[rustc_nounwind]
415+
#[cfg(not(bootstrap))]
416+
pub unsafe fn atomic_load<T: Copy, const ORD: AtomicOrdering>(src: *const T) -> T;
394417
/// Loads the current value of the pointer.
395418
/// `T` must be an integer or pointer type.
396419
///
@@ -399,6 +422,7 @@ pub unsafe fn atomic_cxchgweak_seqcst_seqcst<T: Copy>(dst: *mut T, old: T, src:
399422
/// [`Ordering::SeqCst`] as the `order`. For example, [`AtomicBool::load`].
400423
#[rustc_intrinsic]
401424
#[rustc_nounwind]
425+
#[cfg(bootstrap)]
402426
pub unsafe fn atomic_load_seqcst<T: Copy>(src: *const T) -> T;
403427
/// Loads the current value of the pointer.
404428
/// `T` must be an integer or pointer type.
@@ -408,6 +432,7 @@ pub unsafe fn atomic_load_seqcst<T: Copy>(src: *const T) -> T;
408432
/// [`Ordering::Acquire`] as the `order`. For example, [`AtomicBool::load`].
409433
#[rustc_intrinsic]
410434
#[rustc_nounwind]
435+
#[cfg(bootstrap)]
411436
pub unsafe fn atomic_load_acquire<T: Copy>(src: *const T) -> T;
412437
/// Loads the current value of the pointer.
413438
/// `T` must be an integer or pointer type.
@@ -417,6 +442,7 @@ pub unsafe fn atomic_load_acquire<T: Copy>(src: *const T) -> T;
417442
/// [`Ordering::Relaxed`] as the `order`. For example, [`AtomicBool::load`].
418443
#[rustc_intrinsic]
419444
#[rustc_nounwind]
445+
#[cfg(bootstrap)]
420446
pub unsafe fn atomic_load_relaxed<T: Copy>(src: *const T) -> T;
421447

422448
/// Stores the value at the specified memory location.

‎library/core/src/sync/atomic.rs‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3822,6 +3822,7 @@ unsafe fn atomic_store<T: Copy>(dst: *mut T, val: T, order: Ordering) {
38223822

38233823
#[inline]
38243824
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
3825+
#[cfg(bootstrap)]
38253826
unsafe fn atomic_load<T: Copy>(dst: *const T, order: Ordering) -> T {
38263827
// SAFETY: the caller must uphold the safety contract for `atomic_load`.
38273828
unsafe {
@@ -3835,6 +3836,23 @@ unsafe fn atomic_load<T: Copy>(dst: *const T, order: Ordering) -> T {
38353836
}
38363837
}
38373838

3839+
#[inline]
3840+
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
3841+
#[cfg(not(bootstrap))]
3842+
unsafe fn atomic_load<T: Copy>(dst: *const T, order: Ordering) -> T {
3843+
use intrinsics::AtomicOrdering;
3844+
// SAFETY: the caller must uphold the safety contract for `atomic_load`.
3845+
unsafe {
3846+
match order {
3847+
Relaxed => intrinsics::atomic_load::<T, { AtomicOrdering::Relaxed }>(dst),
3848+
Acquire => intrinsics::atomic_load::<T, { AtomicOrdering::Acquire }>(dst),
3849+
SeqCst => intrinsics::atomic_load::<T, { AtomicOrdering::SeqCst }>(dst),
3850+
Release => panic!("there is no such thing as a release load"),
3851+
AcqRel => panic!("there is no such thing as an acquire-release load"),
3852+
}
3853+
}
3854+
}
3855+
38383856
#[inline]
38393857
#[cfg(target_has_atomic)]
38403858
#[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces

0 commit comments

Comments
 (0)