1//===- InstCombineAtomicRMW.cpp -------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the visit functions for atomic rmw instructions.
10//
11//===----------------------------------------------------------------------===//
12
13#include "InstCombineInternal.h"
14#include "llvm/IR/Instructions.h"
15
16using namespace llvm;
17
18/// Return true if and only if the given instruction does not modify the memory
19/// location referenced. Note that an idemptent atomicrmw may still have
20/// ordering effects on nearby instructions, or be volatile.
21/// TODO: Common w/ the version in AtomicExpandPass, and change the term used.
22/// Idemptotent is confusing in this context.
23static bool isIdempotentRMW(AtomicRMWInst &RMWI) {
24 if (auto CF = dyn_cast<ConstantFP>(Val: RMWI.getValOperand()))
25 switch(RMWI.getOperation()) {
26 case AtomicRMWInst::FAdd: // -0.0
27 return CF->isZero() && CF->isNegative();
28 case AtomicRMWInst::FSub: // +0.0
29 return CF->isZero() && !CF->isNegative();
30 default:
31 return false;
32 };
33
34 auto C = dyn_cast<ConstantInt>(Val: RMWI.getValOperand());
35 if(!C)
36 return false;
37
38 switch(RMWI.getOperation()) {
39 case AtomicRMWInst::Add:
40 case AtomicRMWInst::Sub:
41 case AtomicRMWInst::Or:
42 case AtomicRMWInst::Xor:
43 return C->isZero();
44 case AtomicRMWInst::And:
45 return C->isMinusOne();
46 case AtomicRMWInst::Min:
47 return C->isMaxValue(IsSigned: true);
48 case AtomicRMWInst::Max:
49 return C->isMinValue(IsSigned: true);
50 case AtomicRMWInst::UMin:
51 return C->isMaxValue(IsSigned: false);
52 case AtomicRMWInst::UMax:
53 return C->isMinValue(IsSigned: false);
54 default:
55 return false;
56 }
57}
58
59/// Return true if the given instruction always produces a value in memory
60/// equivalent to its value operand.
61static bool isSaturating(AtomicRMWInst &RMWI) {
62 if (auto CF = dyn_cast<ConstantFP>(Val: RMWI.getValOperand()))
63 switch (RMWI.getOperation()) {
64 case AtomicRMWInst::FMax:
65 // maxnum(x, +inf) -> +inf
66 return !CF->isNegative() && CF->isInfinity();
67 case AtomicRMWInst::FMin:
68 // minnum(x, -inf) -> +inf
69 return CF->isNegative() && CF->isInfinity();
70 case AtomicRMWInst::FAdd:
71 case AtomicRMWInst::FSub:
72 return CF->isNaN();
73 default:
74 return false;
75 };
76
77 auto C = dyn_cast<ConstantInt>(Val: RMWI.getValOperand());
78 if(!C)
79 return false;
80
81 switch(RMWI.getOperation()) {
82 default:
83 return false;
84 case AtomicRMWInst::Xchg:
85 return true;
86 case AtomicRMWInst::Or:
87 return C->isAllOnesValue();
88 case AtomicRMWInst::And:
89 return C->isZero();
90 case AtomicRMWInst::Min:
91 return C->isMinValue(IsSigned: true);
92 case AtomicRMWInst::Max:
93 return C->isMaxValue(IsSigned: true);
94 case AtomicRMWInst::UMin:
95 return C->isMinValue(IsSigned: false);
96 case AtomicRMWInst::UMax:
97 return C->isMaxValue(IsSigned: false);
98 };
99}
100
101Instruction *InstCombinerImpl::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
102
103 // Volatile RMWs perform a load and a store, we cannot replace this by just a
104 // load or just a store. We chose not to canonicalize out of general paranoia
105 // about user expectations around volatile.
106 if (RMWI.isVolatile())
107 return nullptr;
108
109 // Any atomicrmw op which produces a known result in memory can be
110 // replaced w/an atomicrmw xchg.
111 if (isSaturating(RMWI) &&
112 RMWI.getOperation() != AtomicRMWInst::Xchg) {
113 RMWI.setOperation(AtomicRMWInst::Xchg);
114 return &RMWI;
115 }
116
117 assert(RMWI.getOrdering() != AtomicOrdering::NotAtomic &&
118 RMWI.getOrdering() != AtomicOrdering::Unordered &&
119 "AtomicRMWs don't make sense with Unordered or NotAtomic");
120
121 if (!isIdempotentRMW(RMWI))
122 return nullptr;
123
124 // We chose to canonicalize all idempotent operations to an single
125 // operation code and constant. This makes it easier for the rest of the
126 // optimizer to match easily. The choices of or w/0 and fadd w/-0.0 are
127 // arbitrary.
128 if (RMWI.getType()->isIntegerTy() &&
129 RMWI.getOperation() != AtomicRMWInst::Or) {
130 RMWI.setOperation(AtomicRMWInst::Or);
131 return replaceOperand(I&: RMWI, OpNum: 1, V: ConstantInt::get(Ty: RMWI.getType(), V: 0));
132 } else if (RMWI.getType()->isFloatingPointTy() &&
133 RMWI.getOperation() != AtomicRMWInst::FAdd) {
134 RMWI.setOperation(AtomicRMWInst::FAdd);
135 return replaceOperand(I&: RMWI, OpNum: 1, V: ConstantFP::getNegativeZero(Ty: RMWI.getType()));
136 }
137
138 return nullptr;
139}
140