1//===- InstCombineAtomicRMW.cpp -------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the visit functions for atomic rmw instructions.
10//
11//===----------------------------------------------------------------------===//
12
13#include "InstCombineInternal.h"
14#include "llvm/IR/Instructions.h"
15
16using namespace llvm;
17using namespace PatternMatch;
18
19/// Return true if and only if the given instruction does not modify the memory
20/// location referenced. Note that an idemptent atomicrmw may still have
21/// ordering effects on nearby instructions, or be volatile.
22/// TODO: Common w/ the version in AtomicExpandPass, and change the term used.
23/// Idemptotent is confusing in this context.
24static bool isIdempotentRMW(AtomicRMWInst &RMWI) {
25 if (auto CF = dyn_cast<ConstantFP>(Val: RMWI.getValOperand()))
26 switch(RMWI.getOperation()) {
27 case AtomicRMWInst::FAdd: // -0.0
28 return CF->isZero() && CF->isNegative();
29 case AtomicRMWInst::FSub: // +0.0
30 return CF->isZero() && !CF->isNegative();
31 default:
32 return false;
33 };
34
35 auto C = dyn_cast<ConstantInt>(Val: RMWI.getValOperand());
36 if(!C)
37 return false;
38
39 switch(RMWI.getOperation()) {
40 case AtomicRMWInst::Add:
41 case AtomicRMWInst::Sub:
42 case AtomicRMWInst::Or:
43 case AtomicRMWInst::Xor:
44 return C->isZero();
45 case AtomicRMWInst::And:
46 return C->isMinusOne();
47 case AtomicRMWInst::Min:
48 return C->isMaxValue(IsSigned: true);
49 case AtomicRMWInst::Max:
50 return C->isMinValue(IsSigned: true);
51 case AtomicRMWInst::UMin:
52 return C->isMaxValue(IsSigned: false);
53 case AtomicRMWInst::UMax:
54 return C->isMinValue(IsSigned: false);
55 default:
56 return false;
57 }
58}
59
60/// Return true if the given instruction always produces a value in memory
61/// equivalent to its value operand.
62static bool isSaturating(AtomicRMWInst &RMWI) {
63 if (auto CF = dyn_cast<ConstantFP>(Val: RMWI.getValOperand()))
64 switch (RMWI.getOperation()) {
65 case AtomicRMWInst::FMax:
66 // maxnum(x, +inf) -> +inf
67 return !CF->isNegative() && CF->isInfinity();
68 case AtomicRMWInst::FMin:
69 // minnum(x, -inf) -> +inf
70 return CF->isNegative() && CF->isInfinity();
71 case AtomicRMWInst::FAdd:
72 case AtomicRMWInst::FSub:
73 return CF->isNaN();
74 default:
75 return false;
76 };
77
78 auto C = dyn_cast<ConstantInt>(Val: RMWI.getValOperand());
79 if(!C)
80 return false;
81
82 switch(RMWI.getOperation()) {
83 default:
84 return false;
85 case AtomicRMWInst::Xchg:
86 return true;
87 case AtomicRMWInst::Or:
88 return C->isAllOnesValue();
89 case AtomicRMWInst::And:
90 return C->isZero();
91 case AtomicRMWInst::Min:
92 return C->isMinValue(IsSigned: true);
93 case AtomicRMWInst::Max:
94 return C->isMaxValue(IsSigned: true);
95 case AtomicRMWInst::UMin:
96 return C->isMinValue(IsSigned: false);
97 case AtomicRMWInst::UMax:
98 return C->isMaxValue(IsSigned: false);
99 };
100}
101
102Instruction *InstCombinerImpl::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
103
104 // Volatile RMWs perform a load and a store, we cannot replace this by just a
105 // load or just a store. We chose not to canonicalize out of general paranoia
106 // about user expectations around volatile.
107 if (RMWI.isVolatile())
108 return nullptr;
109
110 // Any atomicrmw op which produces a known result in memory can be
111 // replaced w/an atomicrmw xchg.
112 if (isSaturating(RMWI) &&
113 RMWI.getOperation() != AtomicRMWInst::Xchg) {
114 RMWI.setOperation(AtomicRMWInst::Xchg);
115 return &RMWI;
116 }
117
118 assert(RMWI.getOrdering() != AtomicOrdering::NotAtomic &&
119 RMWI.getOrdering() != AtomicOrdering::Unordered &&
120 "AtomicRMWs don't make sense with Unordered or NotAtomic");
121
122 // Canonicalize atomicrmw add(ptr, neg(X)) -> atomicrmw sub(ptr, X)
123 // atomicrmw sub(ptr, neg(X)) -> atomicrmw add(ptr, X)
124 // old + (-X) == old - X and old - (-X) == old + X; the returned old value
125 // is identical in both cases. We match strictly on `sub 0, X` (negation) to
126 // avoid infinite loops: a general negation of `sub A, B` yields `sub B, A`,
127 // which would infinitely be negated back on the next iteration.
128 auto Op = RMWI.getOperation();
129 if (Op == AtomicRMWInst::Add || Op == AtomicRMWInst::Sub) {
130 Value *Val = RMWI.getValOperand();
131 Value *X;
132 if (match(V: Val, P: m_Neg(V: m_Value(V&: X)))) {
133 RMWI.setOperation(Op == AtomicRMWInst::Add ? AtomicRMWInst::Sub
134 : AtomicRMWInst::Add);
135 return replaceOperand(I&: RMWI, OpNum: 1, V: X);
136 }
137 }
138
139 if (!isIdempotentRMW(RMWI))
140 return nullptr;
141
142 // We chose to canonicalize all idempotent operations to an single
143 // operation code and constant. This makes it easier for the rest of the
144 // optimizer to match easily. The choices of or w/0 and fadd w/-0.0 are
145 // arbitrary.
146 if (RMWI.getType()->isIntegerTy() &&
147 RMWI.getOperation() != AtomicRMWInst::Or) {
148 RMWI.setOperation(AtomicRMWInst::Or);
149 return replaceOperand(I&: RMWI, OpNum: 1, V: ConstantInt::get(Ty: RMWI.getType(), V: 0));
150 } else if (RMWI.getType()->isFloatingPointTy() &&
151 RMWI.getOperation() != AtomicRMWInst::FAdd) {
152 RMWI.setOperation(AtomicRMWInst::FAdd);
153 return replaceOperand(I&: RMWI, OpNum: 1, V: ConstantFP::getNegativeZero(Ty: RMWI.getType()));
154 }
155
156 return nullptr;
157}
158