1//===- LowerAtomic.cpp - Lower atomic intrinsics --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass lowers atomic intrinsics to non-atomic form for use in a known
10// non-preemptible environment.
11//
12//===----------------------------------------------------------------------===//
13
14#include "llvm/Transforms/Utils/LowerAtomic.h"
15#include "llvm/IR/Function.h"
16#include "llvm/IR/IRBuilder.h"
17
18using namespace llvm;
19
20#define DEBUG_TYPE "loweratomic"
21
22bool llvm::lowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) {
23 IRBuilder<> Builder(CXI);
24 Value *Ptr = CXI->getPointerOperand();
25 Value *Cmp = CXI->getCompareOperand();
26 Value *Val = CXI->getNewValOperand();
27
28 LoadInst *Orig = Builder.CreateLoad(Ty: Val->getType(), Ptr);
29 Value *Equal = Builder.CreateICmpEQ(LHS: Orig, RHS: Cmp);
30 Value *Res = Builder.CreateSelect(C: Equal, True: Val, False: Orig);
31 Builder.CreateStore(Val: Res, Ptr);
32
33 Res = Builder.CreateInsertValue(Agg: PoisonValue::get(T: CXI->getType()), Val: Orig, Idxs: 0);
34 Res = Builder.CreateInsertValue(Agg: Res, Val: Equal, Idxs: 1);
35
36 CXI->replaceAllUsesWith(V: Res);
37 CXI->eraseFromParent();
38 return true;
39}
40
41Value *llvm::buildAtomicRMWValue(AtomicRMWInst::BinOp Op,
42 IRBuilderBase &Builder, Value *Loaded,
43 Value *Val) {
44 Value *NewVal;
45 switch (Op) {
46 case AtomicRMWInst::Xchg:
47 return Val;
48 case AtomicRMWInst::Add:
49 return Builder.CreateAdd(LHS: Loaded, RHS: Val, Name: "new");
50 case AtomicRMWInst::Sub:
51 return Builder.CreateSub(LHS: Loaded, RHS: Val, Name: "new");
52 case AtomicRMWInst::And:
53 return Builder.CreateAnd(LHS: Loaded, RHS: Val, Name: "new");
54 case AtomicRMWInst::Nand:
55 return Builder.CreateNot(V: Builder.CreateAnd(LHS: Loaded, RHS: Val), Name: "new");
56 case AtomicRMWInst::Or:
57 return Builder.CreateOr(LHS: Loaded, RHS: Val, Name: "new");
58 case AtomicRMWInst::Xor:
59 return Builder.CreateXor(LHS: Loaded, RHS: Val, Name: "new");
60 case AtomicRMWInst::Max:
61 NewVal = Builder.CreateICmpSGT(LHS: Loaded, RHS: Val);
62 return Builder.CreateSelect(C: NewVal, True: Loaded, False: Val, Name: "new");
63 case AtomicRMWInst::Min:
64 NewVal = Builder.CreateICmpSLE(LHS: Loaded, RHS: Val);
65 return Builder.CreateSelect(C: NewVal, True: Loaded, False: Val, Name: "new");
66 case AtomicRMWInst::UMax:
67 NewVal = Builder.CreateICmpUGT(LHS: Loaded, RHS: Val);
68 return Builder.CreateSelect(C: NewVal, True: Loaded, False: Val, Name: "new");
69 case AtomicRMWInst::UMin:
70 NewVal = Builder.CreateICmpULE(LHS: Loaded, RHS: Val);
71 return Builder.CreateSelect(C: NewVal, True: Loaded, False: Val, Name: "new");
72 case AtomicRMWInst::FAdd:
73 return Builder.CreateFAdd(L: Loaded, R: Val, Name: "new");
74 case AtomicRMWInst::FSub:
75 return Builder.CreateFSub(L: Loaded, R: Val, Name: "new");
76 case AtomicRMWInst::FMax:
77 return Builder.CreateMaxNum(LHS: Loaded, RHS: Val);
78 case AtomicRMWInst::FMin:
79 return Builder.CreateMinNum(LHS: Loaded, RHS: Val);
80 case AtomicRMWInst::UIncWrap: {
81 Constant *One = ConstantInt::get(Ty: Loaded->getType(), V: 1);
82 Value *Inc = Builder.CreateAdd(LHS: Loaded, RHS: One);
83 Value *Cmp = Builder.CreateICmpUGE(LHS: Loaded, RHS: Val);
84 Constant *Zero = ConstantInt::get(Ty: Loaded->getType(), V: 0);
85 return Builder.CreateSelect(C: Cmp, True: Zero, False: Inc, Name: "new");
86 }
87 case AtomicRMWInst::UDecWrap: {
88 Constant *Zero = ConstantInt::get(Ty: Loaded->getType(), V: 0);
89 Constant *One = ConstantInt::get(Ty: Loaded->getType(), V: 1);
90
91 Value *Dec = Builder.CreateSub(LHS: Loaded, RHS: One);
92 Value *CmpEq0 = Builder.CreateICmpEQ(LHS: Loaded, RHS: Zero);
93 Value *CmpOldGtVal = Builder.CreateICmpUGT(LHS: Loaded, RHS: Val);
94 Value *Or = Builder.CreateOr(LHS: CmpEq0, RHS: CmpOldGtVal);
95 return Builder.CreateSelect(C: Or, True: Val, False: Dec, Name: "new");
96 }
97 default:
98 llvm_unreachable("Unknown atomic op");
99 }
100}
101
102bool llvm::lowerAtomicRMWInst(AtomicRMWInst *RMWI) {
103 IRBuilder<> Builder(RMWI);
104 Builder.setIsFPConstrained(
105 RMWI->getFunction()->hasFnAttribute(Kind: Attribute::StrictFP));
106
107 Value *Ptr = RMWI->getPointerOperand();
108 Value *Val = RMWI->getValOperand();
109
110 LoadInst *Orig = Builder.CreateLoad(Ty: Val->getType(), Ptr);
111 Value *Res = buildAtomicRMWValue(Op: RMWI->getOperation(), Builder, Loaded: Orig, Val);
112 Builder.CreateStore(Val: Res, Ptr);
113 RMWI->replaceAllUsesWith(V: Orig);
114 RMWI->eraseFromParent();
115 return true;
116}
117