1//===- LowerAtomic.cpp - Lower atomic intrinsics --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass lowers atomic intrinsics to non-atomic form for use in a known
10// non-preemptible environment.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/Utils/LowerAtomic.h"
15#include "llvm/IR/Function.h"
16#include "llvm/IR/IRBuilder.h"
17
18using namespace llvm;
19
20#define DEBUG_TYPE "loweratomic"
21
22bool llvm::lowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) {
23 IRBuilder<> Builder(CXI);
24 Value *Ptr = CXI->getPointerOperand();
25 Value *Cmp = CXI->getCompareOperand();
26 Value *Val = CXI->getNewValOperand();
27
28 auto [Orig, Equal] = buildCmpXchgValue(Builder, Ptr, Cmp, Val,
29 Alignment: CXI->getAlign(), IsVolatile: CXI->isVolatile());
30
31 Value *Res =
32 Builder.CreateInsertValue(Agg: PoisonValue::get(T: CXI->getType()), Val: Orig, Idxs: 0);
33 Res = Builder.CreateInsertValue(Agg: Res, Val: Equal, Idxs: 1);
34
35 CXI->replaceAllUsesWith(V: Res);
36 CXI->eraseFromParent();
37 return true;
38}
39
40std::pair<Value *, Value *> llvm::buildCmpXchgValue(IRBuilderBase &Builder,
41 Value *Ptr, Value *Cmp,
42 Value *Val, Align Alignment,
43 bool IsVolatile) {
44 LoadInst *Orig =
45 Builder.CreateAlignedLoad(Ty: Val->getType(), Ptr, Align: Alignment, isVolatile: IsVolatile);
46 Value *Equal = Builder.CreateICmpEQ(LHS: Orig, RHS: Cmp);
47 // We have no idea what the probability of the value in memory being equal to
48 // the comparison value is without additional VP metadata, so explicitly mark
49 // it unknown.
50 Value *Res =
51 Builder.CreateSelectWithUnknownProfile(C: Equal, True: Val, False: Orig, DEBUG_TYPE);
52 Builder.CreateAlignedStore(Val: Res, Ptr, Align: Alignment, isVolatile: IsVolatile);
53
54 return {Orig, Equal};
55}
56
57Value *llvm::buildAtomicRMWValue(AtomicRMWInst::BinOp Op,
58 IRBuilderBase &Builder, Value *Loaded,
59 Value *Val) {
60 Value *NewVal;
61 switch (Op) {
62 case AtomicRMWInst::Xchg:
63 return Val;
64 case AtomicRMWInst::Add:
65 return Builder.CreateAdd(LHS: Loaded, RHS: Val, Name: "new");
66 case AtomicRMWInst::Sub:
67 return Builder.CreateSub(LHS: Loaded, RHS: Val, Name: "new");
68 case AtomicRMWInst::And:
69 return Builder.CreateAnd(LHS: Loaded, RHS: Val, Name: "new");
70 case AtomicRMWInst::Nand:
71 return Builder.CreateNot(V: Builder.CreateAnd(LHS: Loaded, RHS: Val), Name: "new");
72 case AtomicRMWInst::Or:
73 return Builder.CreateOr(LHS: Loaded, RHS: Val, Name: "new");
74 case AtomicRMWInst::Xor:
75 return Builder.CreateXor(LHS: Loaded, RHS: Val, Name: "new");
76 case AtomicRMWInst::Max:
77 NewVal = Builder.CreateICmpSGT(LHS: Loaded, RHS: Val);
78 return Builder.CreateSelect(C: NewVal, True: Loaded, False: Val, Name: "new");
79 case AtomicRMWInst::Min:
80 NewVal = Builder.CreateICmpSLE(LHS: Loaded, RHS: Val);
81 return Builder.CreateSelect(C: NewVal, True: Loaded, False: Val, Name: "new");
82 case AtomicRMWInst::UMax:
83 NewVal = Builder.CreateICmpUGT(LHS: Loaded, RHS: Val);
84 return Builder.CreateSelect(C: NewVal, True: Loaded, False: Val, Name: "new");
85 case AtomicRMWInst::UMin:
86 NewVal = Builder.CreateICmpULE(LHS: Loaded, RHS: Val);
87 return Builder.CreateSelect(C: NewVal, True: Loaded, False: Val, Name: "new");
88 case AtomicRMWInst::FAdd:
89 return Builder.CreateFAdd(L: Loaded, R: Val, Name: "new");
90 case AtomicRMWInst::FSub:
91 return Builder.CreateFSub(L: Loaded, R: Val, Name: "new");
92 case AtomicRMWInst::FMax:
93 return Builder.CreateMaxNum(LHS: Loaded, RHS: Val);
94 case AtomicRMWInst::FMin:
95 return Builder.CreateMinNum(LHS: Loaded, RHS: Val);
96 case AtomicRMWInst::FMaximum:
97 return Builder.CreateMaximum(LHS: Loaded, RHS: Val);
98 case AtomicRMWInst::FMinimum:
99 return Builder.CreateMinimum(LHS: Loaded, RHS: Val);
100 case AtomicRMWInst::FMaximumNum:
101 return Builder.CreateMaximumNum(LHS: Loaded, RHS: Val);
102 case AtomicRMWInst::FMinimumNum:
103 return Builder.CreateMinimumNum(LHS: Loaded, RHS: Val);
104 case AtomicRMWInst::UIncWrap: {
105 Constant *One = ConstantInt::get(Ty: Loaded->getType(), V: 1);
106 Value *Inc = Builder.CreateAdd(LHS: Loaded, RHS: One);
107 Value *Cmp = Builder.CreateICmpUGE(LHS: Loaded, RHS: Val);
108 Constant *Zero = ConstantInt::get(Ty: Loaded->getType(), V: 0);
109 return Builder.CreateSelect(C: Cmp, True: Zero, False: Inc, Name: "new");
110 }
111 case AtomicRMWInst::UDecWrap: {
112 Constant *Zero = ConstantInt::get(Ty: Loaded->getType(), V: 0);
113 Constant *One = ConstantInt::get(Ty: Loaded->getType(), V: 1);
114
115 Value *Dec = Builder.CreateSub(LHS: Loaded, RHS: One);
116 Value *CmpEq0 = Builder.CreateICmpEQ(LHS: Loaded, RHS: Zero);
117 Value *CmpOldGtVal = Builder.CreateICmpUGT(LHS: Loaded, RHS: Val);
118 Value *Or = Builder.CreateOr(LHS: CmpEq0, RHS: CmpOldGtVal);
119 return Builder.CreateSelect(C: Or, True: Val, False: Dec, Name: "new");
120 }
121 case AtomicRMWInst::USubCond: {
122 Value *Cmp = Builder.CreateICmpUGE(LHS: Loaded, RHS: Val);
123 Value *Sub = Builder.CreateSub(LHS: Loaded, RHS: Val);
124 return Builder.CreateSelect(C: Cmp, True: Sub, False: Loaded, Name: "new");
125 }
126 case AtomicRMWInst::USubSat:
127 return Builder.CreateIntrinsic(ID: Intrinsic::usub_sat, OverloadTypes: Loaded->getType(),
128 Args: {Loaded, Val}, FMFSource: nullptr, Name: "new");
129 default:
130 llvm_unreachable("Unknown atomic op");
131 }
132}
133
134bool llvm::lowerAtomicRMWInst(AtomicRMWInst *RMWI) {
135 IRBuilder<> Builder(RMWI);
136 Builder.setIsFPConstrained(
137 RMWI->getFunction()->hasFnAttribute(Kind: Attribute::StrictFP));
138
139 Value *Ptr = RMWI->getPointerOperand();
140 Value *Val = RMWI->getValOperand();
141
142 LoadInst *Orig = Builder.CreateLoad(Ty: Val->getType(), Ptr, isVolatile: RMWI->isVolatile());
143 Value *Res = buildAtomicRMWValue(Op: RMWI->getOperation(), Builder, Loaded: Orig, Val);
144 Builder.CreateStore(Val: Res, Ptr)->setVolatile(RMWI->isVolatile());
145 RMWI->replaceAllUsesWith(V: Orig);
146 RMWI->eraseFromParent();
147 return true;
148}
149