1//===- LowerAtomic.cpp - Lower atomic intrinsics --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass lowers atomic intrinsics to non-atomic form for use in a known
10// non-preemptible environment.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/Utils/LowerAtomic.h"
15#include "llvm/IR/Function.h"
16#include "llvm/IR/IRBuilder.h"
17
18using namespace llvm;
19
20#define DEBUG_TYPE "loweratomic"
21
22bool llvm::lowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) {
23 IRBuilder<> Builder(CXI);
24 Value *Ptr = CXI->getPointerOperand();
25 Value *Cmp = CXI->getCompareOperand();
26 Value *Val = CXI->getNewValOperand();
27
28 auto [Orig, Equal] =
29 buildCmpXchgValue(Builder, Ptr, Cmp, Val, Alignment: CXI->getAlign());
30
31 Value *Res =
32 Builder.CreateInsertValue(Agg: PoisonValue::get(T: CXI->getType()), Val: Orig, Idxs: 0);
33 Res = Builder.CreateInsertValue(Agg: Res, Val: Equal, Idxs: 1);
34
35 CXI->replaceAllUsesWith(V: Res);
36 CXI->eraseFromParent();
37 return true;
38}
39
40std::pair<Value *, Value *> llvm::buildCmpXchgValue(IRBuilderBase &Builder,
41 Value *Ptr, Value *Cmp,
42 Value *Val,
43 Align Alignment) {
44 LoadInst *Orig = Builder.CreateAlignedLoad(Ty: Val->getType(), Ptr, Align: Alignment);
45 Value *Equal = Builder.CreateICmpEQ(LHS: Orig, RHS: Cmp);
46 Value *Res = Builder.CreateSelect(C: Equal, True: Val, False: Orig);
47 Builder.CreateAlignedStore(Val: Res, Ptr, Align: Alignment);
48
49 return {Orig, Equal};
50}
51
52Value *llvm::buildAtomicRMWValue(AtomicRMWInst::BinOp Op,
53 IRBuilderBase &Builder, Value *Loaded,
54 Value *Val) {
55 Value *NewVal;
56 switch (Op) {
57 case AtomicRMWInst::Xchg:
58 return Val;
59 case AtomicRMWInst::Add:
60 return Builder.CreateAdd(LHS: Loaded, RHS: Val, Name: "new");
61 case AtomicRMWInst::Sub:
62 return Builder.CreateSub(LHS: Loaded, RHS: Val, Name: "new");
63 case AtomicRMWInst::And:
64 return Builder.CreateAnd(LHS: Loaded, RHS: Val, Name: "new");
65 case AtomicRMWInst::Nand:
66 return Builder.CreateNot(V: Builder.CreateAnd(LHS: Loaded, RHS: Val), Name: "new");
67 case AtomicRMWInst::Or:
68 return Builder.CreateOr(LHS: Loaded, RHS: Val, Name: "new");
69 case AtomicRMWInst::Xor:
70 return Builder.CreateXor(LHS: Loaded, RHS: Val, Name: "new");
71 case AtomicRMWInst::Max:
72 NewVal = Builder.CreateICmpSGT(LHS: Loaded, RHS: Val);
73 return Builder.CreateSelect(C: NewVal, True: Loaded, False: Val, Name: "new");
74 case AtomicRMWInst::Min:
75 NewVal = Builder.CreateICmpSLE(LHS: Loaded, RHS: Val);
76 return Builder.CreateSelect(C: NewVal, True: Loaded, False: Val, Name: "new");
77 case AtomicRMWInst::UMax:
78 NewVal = Builder.CreateICmpUGT(LHS: Loaded, RHS: Val);
79 return Builder.CreateSelect(C: NewVal, True: Loaded, False: Val, Name: "new");
80 case AtomicRMWInst::UMin:
81 NewVal = Builder.CreateICmpULE(LHS: Loaded, RHS: Val);
82 return Builder.CreateSelect(C: NewVal, True: Loaded, False: Val, Name: "new");
83 case AtomicRMWInst::FAdd:
84 return Builder.CreateFAdd(L: Loaded, R: Val, Name: "new");
85 case AtomicRMWInst::FSub:
86 return Builder.CreateFSub(L: Loaded, R: Val, Name: "new");
87 case AtomicRMWInst::FMax:
88 return Builder.CreateMaxNum(LHS: Loaded, RHS: Val);
89 case AtomicRMWInst::FMin:
90 return Builder.CreateMinNum(LHS: Loaded, RHS: Val);
91 case AtomicRMWInst::FMaximum:
92 return Builder.CreateMaximum(LHS: Loaded, RHS: Val);
93 case AtomicRMWInst::FMinimum:
94 return Builder.CreateMinimum(LHS: Loaded, RHS: Val);
95 case AtomicRMWInst::FMaximumNum:
96 return Builder.CreateMaximumNum(LHS: Loaded, RHS: Val);
97 case AtomicRMWInst::FMinimumNum:
98 return Builder.CreateMinimumNum(LHS: Loaded, RHS: Val);
99 case AtomicRMWInst::UIncWrap: {
100 Constant *One = ConstantInt::get(Ty: Loaded->getType(), V: 1);
101 Value *Inc = Builder.CreateAdd(LHS: Loaded, RHS: One);
102 Value *Cmp = Builder.CreateICmpUGE(LHS: Loaded, RHS: Val);
103 Constant *Zero = ConstantInt::get(Ty: Loaded->getType(), V: 0);
104 return Builder.CreateSelect(C: Cmp, True: Zero, False: Inc, Name: "new");
105 }
106 case AtomicRMWInst::UDecWrap: {
107 Constant *Zero = ConstantInt::get(Ty: Loaded->getType(), V: 0);
108 Constant *One = ConstantInt::get(Ty: Loaded->getType(), V: 1);
109
110 Value *Dec = Builder.CreateSub(LHS: Loaded, RHS: One);
111 Value *CmpEq0 = Builder.CreateICmpEQ(LHS: Loaded, RHS: Zero);
112 Value *CmpOldGtVal = Builder.CreateICmpUGT(LHS: Loaded, RHS: Val);
113 Value *Or = Builder.CreateOr(LHS: CmpEq0, RHS: CmpOldGtVal);
114 return Builder.CreateSelect(C: Or, True: Val, False: Dec, Name: "new");
115 }
116 case AtomicRMWInst::USubCond: {
117 Value *Cmp = Builder.CreateICmpUGE(LHS: Loaded, RHS: Val);
118 Value *Sub = Builder.CreateSub(LHS: Loaded, RHS: Val);
119 return Builder.CreateSelect(C: Cmp, True: Sub, False: Loaded, Name: "new");
120 }
121 case AtomicRMWInst::USubSat:
122 return Builder.CreateIntrinsic(ID: Intrinsic::usub_sat, Types: Loaded->getType(),
123 Args: {Loaded, Val}, FMFSource: nullptr, Name: "new");
124 default:
125 llvm_unreachable("Unknown atomic op");
126 }
127}
128
129bool llvm::lowerAtomicRMWInst(AtomicRMWInst *RMWI) {
130 IRBuilder<> Builder(RMWI);
131 Builder.setIsFPConstrained(
132 RMWI->getFunction()->hasFnAttribute(Kind: Attribute::StrictFP));
133
134 Value *Ptr = RMWI->getPointerOperand();
135 Value *Val = RMWI->getValOperand();
136
137 LoadInst *Orig = Builder.CreateLoad(Ty: Val->getType(), Ptr);
138 Value *Res = buildAtomicRMWValue(Op: RMWI->getOperation(), Builder, Loaded: Orig, Val);
139 Builder.CreateStore(Val: Res, Ptr);
140 RMWI->replaceAllUsesWith(V: Orig);
141 RMWI->eraseFromParent();
142 return true;
143}
144