1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains one of the several passes that expand pseudo instructions
10// into target instructions. This pass is run at the last possible moment,
11// avoiding the possibility for other passes to break the requirements for
12// forward progress in the LR/SC block.
13//
14//===----------------------------------------------------------------------===//
15
16#include "RISCV.h"
17#include "RISCVExpandPseudoBase.h"
18#include "RISCVInstrInfo.h"
19#include "RISCVSubtarget.h"
20#include "llvm/CodeGen/LivePhysRegs.h"
21#include "llvm/CodeGen/MachineFunctionPass.h"
22#include "llvm/CodeGen/MachineInstrBuilder.h"
23
24using namespace llvm;
25
26#define RISCV_EXPAND_PSEUDO_ATOMICS_NAME \
27 "RISC-V Pseudo Instruction Expansion - Atomics"
28
29namespace {
30
31class RISCVExpandPseudoAtomicsImpl final : public RISCVExpandPseudoImplBase {
32 bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
33 MachineBasicBlock::iterator &NextMBBI) const override;
34
35 bool expandAtomicBinOp(MachineBasicBlock &MBB,
36 MachineBasicBlock::iterator MBBI, AtomicRMWInst::BinOp,
37 bool IsMasked, int Width,
38 MachineBasicBlock::iterator &NextMBBI) const;
39
40 bool expandAtomicMinMaxOp(MachineBasicBlock &MBB,
41 MachineBasicBlock::iterator MBBI,
42 AtomicRMWInst::BinOp, bool IsMasked, int Width,
43 MachineBasicBlock::iterator &NextMBBI) const;
44
45 bool expandAtomicCmpXchg(MachineBasicBlock &MBB,
46 MachineBasicBlock::iterator MBBI, bool IsMasked,
47 int Width,
48 MachineBasicBlock::iterator &NextMBBI) const;
49
50 unsigned getLRForRMW32(AtomicOrdering Ordering) const;
51 unsigned getSCForRMW32(AtomicOrdering Ordering) const;
52 unsigned getLRForRMW64(AtomicOrdering Ordering) const;
53 unsigned getSCForRMW64(AtomicOrdering Ordering) const;
54 unsigned getLRForRMW(AtomicOrdering Ordering, int Width) const;
55 unsigned getSCForRMW(AtomicOrdering Ordering, int Width) const;
56
57 void doAtomicBinOpExpansion(MachineInstr &MI, MachineBasicBlock *LoopMBB,
58 AtomicRMWInst::BinOp BinOp, int Width) const;
59 void insertMaskedMerge(DebugLoc DL, MachineBasicBlock *MBB, Register DestReg,
60 Register OldValReg, Register NewValReg,
61 Register MaskReg, Register ScratchReg) const;
62 void doMaskedAtomicBinOpExpansion(MachineInstr &MI,
63 MachineBasicBlock *LoopMBB,
64 AtomicRMWInst::BinOp BinOp,
65 int Width) const;
66 void insertSext(DebugLoc DL, MachineBasicBlock *MBB, Register ValReg,
67 Register ShamtReg) const;
68 void doAtomicMinMaxOpExpansion(MachineInstr &MI,
69 MachineBasicBlock *LoopHeadMBB,
70 MachineBasicBlock *LoopIfBodyMBB,
71 MachineBasicBlock *LoopTailMBB,
72 AtomicRMWInst::BinOp BinOp, int Width) const;
73 void doMaskedAtomicMinMaxOpExpansion(MachineInstr &MI,
74 MachineBasicBlock *LoopHeadMBB,
75 MachineBasicBlock *LoopIfBodyMBB,
76 MachineBasicBlock *LoopTailMBB,
77 AtomicRMWInst::BinOp BinOp,
78 int Width) const;
79 bool tryToFoldBNEOnCmpXchgResult(MachineBasicBlock &MBB,
80 MachineBasicBlock::iterator MBBI,
81 Register DestReg, Register CmpValReg,
82 Register MaskReg,
83 MachineBasicBlock *&LoopHeadBNETarget) const;
84};
85
86class RISCVExpandPseudoAtomicsLegacy : public MachineFunctionPass {
87public:
88 static char ID;
89
90 RISCVExpandPseudoAtomicsLegacy() : MachineFunctionPass(ID) {}
91
92 bool runOnMachineFunction(MachineFunction &MF) override {
93 return RISCVExpandPseudoAtomicsImpl().run(MF);
94 }
95
96 StringRef getPassName() const override {
97 return RISCV_EXPAND_PSEUDO_ATOMICS_NAME;
98 }
99};
100
101} // anonymous namespace
102
103bool RISCVExpandPseudoAtomicsImpl::expandMI(
104 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
105 MachineBasicBlock::iterator &NextMBBI) const {
106 // RISCVInstrInfo::getInstSizeInBytes expects that the total size of the
107 // expanded instructions for each pseudo is correct in the Size field of the
108 // tablegen definition for the pseudo.
109 switch (MBBI->getOpcode()) {
110 case RISCV::PseudoAtomicSwap32:
111 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, IsMasked: false, Width: 32,
112 NextMBBI);
113 case RISCV::PseudoAtomicSwap64:
114 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, IsMasked: false, Width: 64,
115 NextMBBI);
116 case RISCV::PseudoAtomicLoadAdd32:
117 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, IsMasked: false, Width: 32,
118 NextMBBI);
119 case RISCV::PseudoAtomicLoadAdd64:
120 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, IsMasked: false, Width: 64,
121 NextMBBI);
122 case RISCV::PseudoAtomicLoadSub32:
123 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, IsMasked: false, Width: 32,
124 NextMBBI);
125 case RISCV::PseudoAtomicLoadSub64:
126 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, IsMasked: false, Width: 64,
127 NextMBBI);
128 case RISCV::PseudoAtomicLoadAnd32:
129 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::And, IsMasked: false, Width: 32,
130 NextMBBI);
131 case RISCV::PseudoAtomicLoadAnd64:
132 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::And, IsMasked: false, Width: 64,
133 NextMBBI);
134 case RISCV::PseudoAtomicLoadOr32:
135 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Or, IsMasked: false, Width: 32, NextMBBI);
136 case RISCV::PseudoAtomicLoadOr64:
137 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Or, IsMasked: false, Width: 64, NextMBBI);
138 case RISCV::PseudoAtomicLoadXor32:
139 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xor, IsMasked: false, Width: 32,
140 NextMBBI);
141 case RISCV::PseudoAtomicLoadXor64:
142 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xor, IsMasked: false, Width: 64,
143 NextMBBI);
144 case RISCV::PseudoAtomicLoadNand32:
145 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, IsMasked: false, Width: 32,
146 NextMBBI);
147 case RISCV::PseudoAtomicLoadNand64:
148 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, IsMasked: false, Width: 64,
149 NextMBBI);
150 case RISCV::PseudoAtomicLoadMin32:
151 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Min, IsMasked: false, Width: 32,
152 NextMBBI);
153 case RISCV::PseudoAtomicLoadMin64:
154 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Min, IsMasked: false, Width: 64,
155 NextMBBI);
156 case RISCV::PseudoAtomicLoadMax32:
157 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, IsMasked: false, Width: 32,
158 NextMBBI);
159 case RISCV::PseudoAtomicLoadMax64:
160 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, IsMasked: false, Width: 64,
161 NextMBBI);
162 case RISCV::PseudoAtomicLoadUMin32:
163 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, IsMasked: false, Width: 32,
164 NextMBBI);
165 case RISCV::PseudoAtomicLoadUMin64:
166 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, IsMasked: false, Width: 64,
167 NextMBBI);
168 case RISCV::PseudoAtomicLoadUMax32:
169 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, IsMasked: false, Width: 32,
170 NextMBBI);
171 case RISCV::PseudoAtomicLoadUMax64:
172 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, IsMasked: false, Width: 64,
173 NextMBBI);
174 case RISCV::PseudoMaskedAtomicSwap32:
175 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, IsMasked: true, Width: 32,
176 NextMBBI);
177 case RISCV::PseudoMaskedAtomicLoadAdd32:
178 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, IsMasked: true, Width: 32, NextMBBI);
179 case RISCV::PseudoMaskedAtomicLoadSub32:
180 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, IsMasked: true, Width: 32, NextMBBI);
181 case RISCV::PseudoMaskedAtomicLoadNand32:
182 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, IsMasked: true, Width: 32,
183 NextMBBI);
184 case RISCV::PseudoMaskedAtomicLoadMax32:
185 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, IsMasked: true, Width: 32,
186 NextMBBI);
187 case RISCV::PseudoMaskedAtomicLoadMin32:
188 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Min, IsMasked: true, Width: 32,
189 NextMBBI);
190 case RISCV::PseudoMaskedAtomicLoadUMax32:
191 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, IsMasked: true, Width: 32,
192 NextMBBI);
193 case RISCV::PseudoMaskedAtomicLoadUMin32:
194 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, IsMasked: true, Width: 32,
195 NextMBBI);
196 case RISCV::PseudoCmpXchg32:
197 return expandAtomicCmpXchg(MBB, MBBI, IsMasked: false, Width: 32, NextMBBI);
198 case RISCV::PseudoCmpXchg64:
199 return expandAtomicCmpXchg(MBB, MBBI, IsMasked: false, Width: 64, NextMBBI);
200 case RISCV::PseudoMaskedCmpXchg32:
201 return expandAtomicCmpXchg(MBB, MBBI, IsMasked: true, Width: 32, NextMBBI);
202 }
203
204 return false;
205}
206
207unsigned
208RISCVExpandPseudoAtomicsImpl::getLRForRMW32(AtomicOrdering Ordering) const {
209 switch (Ordering) {
210 default:
211 llvm_unreachable("Unexpected AtomicOrdering");
212 case AtomicOrdering::Monotonic:
213 return RISCV::LR_W;
214 case AtomicOrdering::Acquire:
215 if (STI->hasStdExtZtso())
216 return RISCV::LR_W;
217 return RISCV::LR_W_AQ;
218 case AtomicOrdering::Release:
219 return RISCV::LR_W;
220 case AtomicOrdering::AcquireRelease:
221 if (STI->hasStdExtZtso())
222 return RISCV::LR_W;
223 return RISCV::LR_W_AQ;
224 case AtomicOrdering::SequentiallyConsistent:
225 return RISCV::LR_W_AQRL;
226 }
227}
228
229unsigned
230RISCVExpandPseudoAtomicsImpl::getSCForRMW32(AtomicOrdering Ordering) const {
231 switch (Ordering) {
232 default:
233 llvm_unreachable("Unexpected AtomicOrdering");
234 case AtomicOrdering::Monotonic:
235 return RISCV::SC_W;
236 case AtomicOrdering::Acquire:
237 return RISCV::SC_W;
238 case AtomicOrdering::Release:
239 if (STI->hasStdExtZtso())
240 return RISCV::SC_W;
241 return RISCV::SC_W_RL;
242 case AtomicOrdering::AcquireRelease:
243 if (STI->hasStdExtZtso())
244 return RISCV::SC_W;
245 return RISCV::SC_W_RL;
246 case AtomicOrdering::SequentiallyConsistent:
247 return RISCV::SC_W_RL;
248 }
249}
250
251unsigned
252RISCVExpandPseudoAtomicsImpl::getLRForRMW64(AtomicOrdering Ordering) const {
253 switch (Ordering) {
254 default:
255 llvm_unreachable("Unexpected AtomicOrdering");
256 case AtomicOrdering::Monotonic:
257 return RISCV::LR_D;
258 case AtomicOrdering::Acquire:
259 if (STI->hasStdExtZtso())
260 return RISCV::LR_D;
261 return RISCV::LR_D_AQ;
262 case AtomicOrdering::Release:
263 return RISCV::LR_D;
264 case AtomicOrdering::AcquireRelease:
265 if (STI->hasStdExtZtso())
266 return RISCV::LR_D;
267 return RISCV::LR_D_AQ;
268 case AtomicOrdering::SequentiallyConsistent:
269 return RISCV::LR_D_AQRL;
270 }
271}
272
273unsigned
274RISCVExpandPseudoAtomicsImpl::getSCForRMW64(AtomicOrdering Ordering) const {
275 switch (Ordering) {
276 default:
277 llvm_unreachable("Unexpected AtomicOrdering");
278 case AtomicOrdering::Monotonic:
279 return RISCV::SC_D;
280 case AtomicOrdering::Acquire:
281 return RISCV::SC_D;
282 case AtomicOrdering::Release:
283 if (STI->hasStdExtZtso())
284 return RISCV::SC_D;
285 return RISCV::SC_D_RL;
286 case AtomicOrdering::AcquireRelease:
287 if (STI->hasStdExtZtso())
288 return RISCV::SC_D;
289 return RISCV::SC_D_RL;
290 case AtomicOrdering::SequentiallyConsistent:
291 return RISCV::SC_D_RL;
292 }
293}
294
295unsigned RISCVExpandPseudoAtomicsImpl::getLRForRMW(AtomicOrdering Ordering,
296 int Width) const {
297 if (Width == 32)
298 return getLRForRMW32(Ordering);
299 if (Width == 64)
300 return getLRForRMW64(Ordering);
301 llvm_unreachable("Unexpected LR width\n");
302}
303
304unsigned RISCVExpandPseudoAtomicsImpl::getSCForRMW(AtomicOrdering Ordering,
305 int Width) const {
306 if (Width == 32)
307 return getSCForRMW32(Ordering);
308 if (Width == 64)
309 return getSCForRMW64(Ordering);
310 llvm_unreachable("Unexpected SC width\n");
311}
312
313void RISCVExpandPseudoAtomicsImpl::doAtomicBinOpExpansion(
314 MachineInstr &MI, MachineBasicBlock *LoopMBB, AtomicRMWInst::BinOp BinOp,
315 int Width) const {
316 DebugLoc DL = MI.getDebugLoc();
317 Register DestReg = MI.getOperand(i: 0).getReg();
318 Register ScratchReg = MI.getOperand(i: 1).getReg();
319 Register AddrReg = MI.getOperand(i: 2).getReg();
320 Register IncrReg = MI.getOperand(i: 3).getReg();
321 AtomicOrdering Ordering =
322 static_cast<AtomicOrdering>(MI.getOperand(i: 4).getImm());
323
324 // .loop:
325 // lr.[w|d] dest, (addr)
326 // binop scratch, dest, val
327 // sc.[w|d] scratch, scratch, (addr)
328 // bnez scratch, loop
329 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: getLRForRMW(Ordering, Width)), DestReg)
330 .addReg(RegNo: AddrReg);
331 switch (BinOp) {
332 default:
333 llvm_unreachable("Unexpected AtomicRMW BinOp");
334 case AtomicRMWInst::Xchg:
335 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::ADDI), DestReg: ScratchReg)
336 .addReg(RegNo: IncrReg)
337 .addImm(Val: 0);
338 break;
339 case AtomicRMWInst::Add:
340 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::ADD), DestReg: ScratchReg)
341 .addReg(RegNo: DestReg)
342 .addReg(RegNo: IncrReg);
343 break;
344 case AtomicRMWInst::Sub:
345 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::SUB), DestReg: ScratchReg)
346 .addReg(RegNo: DestReg)
347 .addReg(RegNo: IncrReg);
348 break;
349 case AtomicRMWInst::And:
350 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::AND), DestReg: ScratchReg)
351 .addReg(RegNo: DestReg)
352 .addReg(RegNo: IncrReg);
353 break;
354 case AtomicRMWInst::Or:
355 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::OR), DestReg: ScratchReg)
356 .addReg(RegNo: DestReg)
357 .addReg(RegNo: IncrReg);
358 break;
359 case AtomicRMWInst::Xor:
360 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::XOR), DestReg: ScratchReg)
361 .addReg(RegNo: DestReg)
362 .addReg(RegNo: IncrReg);
363 break;
364 case AtomicRMWInst::Nand:
365 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::AND), DestReg: ScratchReg)
366 .addReg(RegNo: DestReg)
367 .addReg(RegNo: IncrReg);
368 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::XORI), DestReg: ScratchReg)
369 .addReg(RegNo: ScratchReg)
370 .addImm(Val: -1);
371 break;
372 case AtomicRMWInst::Max:
373 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::MAX), DestReg: ScratchReg)
374 .addReg(RegNo: DestReg)
375 .addReg(RegNo: IncrReg);
376 break;
377 case AtomicRMWInst::Min:
378 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::MIN), DestReg: ScratchReg)
379 .addReg(RegNo: DestReg)
380 .addReg(RegNo: IncrReg);
381 break;
382 case AtomicRMWInst::UMax:
383 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::MAXU), DestReg: ScratchReg)
384 .addReg(RegNo: DestReg)
385 .addReg(RegNo: IncrReg);
386 break;
387 case AtomicRMWInst::UMin:
388 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::MINU), DestReg: ScratchReg)
389 .addReg(RegNo: DestReg)
390 .addReg(RegNo: IncrReg);
391 break;
392 }
393 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: getSCForRMW(Ordering, Width)), DestReg: ScratchReg)
394 .addReg(RegNo: ScratchReg)
395 .addReg(RegNo: AddrReg);
396 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BNE))
397 .addReg(RegNo: ScratchReg)
398 .addReg(RegNo: RISCV::X0)
399 .addMBB(MBB: LoopMBB);
400}
401
402void RISCVExpandPseudoAtomicsImpl::insertMaskedMerge(
403 DebugLoc DL, MachineBasicBlock *MBB, Register DestReg, Register OldValReg,
404 Register NewValReg, Register MaskReg, Register ScratchReg) const {
405 assert(OldValReg != ScratchReg && "OldValReg and ScratchReg must be unique");
406 assert(OldValReg != MaskReg && "OldValReg and MaskReg must be unique");
407 assert(ScratchReg != MaskReg && "ScratchReg and MaskReg must be unique");
408
409 // We select bits from newval and oldval using:
410 // https://graphics.stanford.edu/~seander/bithacks.html#MaskedMerge
411 // r = oldval ^ ((oldval ^ newval) & masktargetdata);
412 // A zero newval reduces this to r = oldval ^ (oldval & masktargetdata).
413 if (NewValReg == RISCV::X0) {
414 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::AND), DestReg: ScratchReg)
415 .addReg(RegNo: OldValReg)
416 .addReg(RegNo: MaskReg);
417 } else {
418 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::XOR), DestReg: ScratchReg)
419 .addReg(RegNo: OldValReg)
420 .addReg(RegNo: NewValReg);
421 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::AND), DestReg: ScratchReg)
422 .addReg(RegNo: ScratchReg)
423 .addReg(RegNo: MaskReg);
424 }
425 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::XOR), DestReg)
426 .addReg(RegNo: OldValReg)
427 .addReg(RegNo: ScratchReg);
428}
429
430void RISCVExpandPseudoAtomicsImpl::doMaskedAtomicBinOpExpansion(
431 MachineInstr &MI, MachineBasicBlock *LoopMBB, AtomicRMWInst::BinOp BinOp,
432 int Width) const {
433 DebugLoc DL = MI.getDebugLoc();
434 assert(Width == 32 && "Should never need to expand masked 64-bit operations");
435 Register DestReg = MI.getOperand(i: 0).getReg();
436 Register ScratchReg = MI.getOperand(i: 1).getReg();
437 Register AddrReg = MI.getOperand(i: 2).getReg();
438 Register IncrReg = MI.getOperand(i: 3).getReg();
439 Register MaskReg = MI.getOperand(i: 4).getReg();
440 AtomicOrdering Ordering =
441 static_cast<AtomicOrdering>(MI.getOperand(i: 5).getImm());
442
443 // .loop:
444 // lr.w destreg, (alignedaddr)
445 // binop scratch, destreg, incr
446 // xor scratch, destreg, scratch
447 // and scratch, scratch, masktargetdata
448 // xor scratch, destreg, scratch
449 // sc.w scratch, scratch, (alignedaddr)
450 // bnez scratch, loop
451 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: getLRForRMW32(Ordering)), DestReg)
452 .addReg(RegNo: AddrReg);
453 switch (BinOp) {
454 default:
455 llvm_unreachable("Unexpected AtomicRMW BinOp");
456 case AtomicRMWInst::Xchg:
457 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::ADDI), DestReg: ScratchReg)
458 .addReg(RegNo: IncrReg)
459 .addImm(Val: 0);
460 break;
461 case AtomicRMWInst::Add:
462 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::ADD), DestReg: ScratchReg)
463 .addReg(RegNo: DestReg)
464 .addReg(RegNo: IncrReg);
465 break;
466 case AtomicRMWInst::Sub:
467 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::SUB), DestReg: ScratchReg)
468 .addReg(RegNo: DestReg)
469 .addReg(RegNo: IncrReg);
470 break;
471 case AtomicRMWInst::Nand:
472 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::AND), DestReg: ScratchReg)
473 .addReg(RegNo: DestReg)
474 .addReg(RegNo: IncrReg);
475 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::XORI), DestReg: ScratchReg)
476 .addReg(RegNo: ScratchReg)
477 .addImm(Val: -1);
478 break;
479 }
480
481 insertMaskedMerge(DL, MBB: LoopMBB, DestReg: ScratchReg, OldValReg: DestReg, NewValReg: ScratchReg, MaskReg,
482 ScratchReg);
483
484 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: getSCForRMW32(Ordering)), DestReg: ScratchReg)
485 .addReg(RegNo: ScratchReg)
486 .addReg(RegNo: AddrReg);
487 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BNE))
488 .addReg(RegNo: ScratchReg)
489 .addReg(RegNo: RISCV::X0)
490 .addMBB(MBB: LoopMBB);
491}
492
493bool RISCVExpandPseudoAtomicsImpl::expandAtomicBinOp(
494 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
495 AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
496 MachineBasicBlock::iterator &NextMBBI) const {
497 MachineInstr &MI = *MBBI;
498 MachineFunction *MF = MBB.getParent();
499 auto LoopMBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
500 auto DoneMBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
501
502 // Insert new MBBs.
503 MF->insert(MBBI: ++MBB.getIterator(), MBB: LoopMBB);
504 MF->insert(MBBI: ++LoopMBB->getIterator(), MBB: DoneMBB);
505
506 // Set up successors and transfer remaining instructions to DoneMBB.
507 LoopMBB->addSuccessor(Succ: LoopMBB);
508 LoopMBB->addSuccessor(Succ: DoneMBB);
509 DoneMBB->splice(Where: DoneMBB->end(), Other: &MBB, From: MI, To: MBB.end());
510 DoneMBB->transferSuccessors(FromMBB: &MBB);
511 MBB.addSuccessor(Succ: LoopMBB);
512
513 if (!IsMasked)
514 doAtomicBinOpExpansion(MI, LoopMBB, BinOp, Width);
515 else
516 doMaskedAtomicBinOpExpansion(MI, LoopMBB, BinOp, Width);
517
518 NextMBBI = MBB.end();
519 MI.eraseFromParent();
520
521 LivePhysRegs LiveRegs;
522 computeAndAddLiveIns(LiveRegs, MBB&: *LoopMBB);
523 computeAndAddLiveIns(LiveRegs, MBB&: *DoneMBB);
524
525 return true;
526}
527
528void RISCVExpandPseudoAtomicsImpl::insertSext(DebugLoc DL,
529 MachineBasicBlock *MBB,
530 Register ValReg,
531 Register ShamtReg) const {
532 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::SLL), DestReg: ValReg)
533 .addReg(RegNo: ValReg)
534 .addReg(RegNo: ShamtReg);
535 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::SRA), DestReg: ValReg)
536 .addReg(RegNo: ValReg)
537 .addReg(RegNo: ShamtReg);
538}
539
540void RISCVExpandPseudoAtomicsImpl::doAtomicMinMaxOpExpansion(
541 MachineInstr &MI, MachineBasicBlock *LoopHeadMBB,
542 MachineBasicBlock *LoopIfBodyMBB, MachineBasicBlock *LoopTailMBB,
543 AtomicRMWInst::BinOp BinOp, int Width) const {
544 DebugLoc DL = MI.getDebugLoc();
545 Register DestReg = MI.getOperand(i: 0).getReg();
546 Register ScratchReg = MI.getOperand(i: 1).getReg();
547 Register AddrReg = MI.getOperand(i: 2).getReg();
548 Register IncrReg = MI.getOperand(i: 3).getReg();
549 AtomicOrdering Ordering =
550 static_cast<AtomicOrdering>(MI.getOperand(i: 4).getImm());
551
552 // .loophead:
553 // lr.[w|d] dest, (addr)
554 // mv scratch, dest
555 // ifnochangeneeded scratch, incr, .looptail
556 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: getLRForRMW(Ordering, Width)), DestReg)
557 .addReg(RegNo: AddrReg);
558 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::ADDI), DestReg: ScratchReg)
559 .addReg(RegNo: DestReg)
560 .addImm(Val: 0);
561 switch (BinOp) {
562 default:
563 llvm_unreachable("Unexpected AtomicRMW BinOp");
564 case AtomicRMWInst::Max: {
565 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BGE))
566 .addReg(RegNo: ScratchReg)
567 .addReg(RegNo: IncrReg)
568 .addMBB(MBB: LoopTailMBB);
569 break;
570 }
571 case AtomicRMWInst::Min: {
572 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BGE))
573 .addReg(RegNo: IncrReg)
574 .addReg(RegNo: ScratchReg)
575 .addMBB(MBB: LoopTailMBB);
576 break;
577 }
578 case AtomicRMWInst::UMax:
579 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BGEU))
580 .addReg(RegNo: ScratchReg)
581 .addReg(RegNo: IncrReg)
582 .addMBB(MBB: LoopTailMBB);
583 break;
584 case AtomicRMWInst::UMin:
585 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BGEU))
586 .addReg(RegNo: IncrReg)
587 .addReg(RegNo: ScratchReg)
588 .addMBB(MBB: LoopTailMBB);
589 break;
590 }
591
592 // .loopifbody:
593 // mv scratch, incr
594 BuildMI(BB: LoopIfBodyMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::ADDI), DestReg: ScratchReg)
595 .addReg(RegNo: IncrReg)
596 .addImm(Val: 0);
597
598 // .looptail:
599 // sc.[w|d] scratch, scratch, (addr)
600 // bnez scratch, loop
601 BuildMI(BB: LoopTailMBB, MIMD: DL, MCID: TII->get(Opcode: getSCForRMW(Ordering, Width)), DestReg: ScratchReg)
602 .addReg(RegNo: ScratchReg)
603 .addReg(RegNo: AddrReg);
604 BuildMI(BB: LoopTailMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BNE))
605 .addReg(RegNo: ScratchReg)
606 .addReg(RegNo: RISCV::X0)
607 .addMBB(MBB: LoopHeadMBB);
608}
609
610void RISCVExpandPseudoAtomicsImpl::doMaskedAtomicMinMaxOpExpansion(
611 MachineInstr &MI, MachineBasicBlock *LoopHeadMBB,
612 MachineBasicBlock *LoopIfBodyMBB, MachineBasicBlock *LoopTailMBB,
613 AtomicRMWInst::BinOp BinOp, int Width) const {
614 DebugLoc DL = MI.getDebugLoc();
615 assert(Width == 32 && "Should never need to expand masked 64-bit operations");
616 Register DestReg = MI.getOperand(i: 0).getReg();
617 Register Scratch1Reg = MI.getOperand(i: 1).getReg();
618 Register Scratch2Reg = MI.getOperand(i: 2).getReg();
619 Register AddrReg = MI.getOperand(i: 3).getReg();
620 Register IncrReg = MI.getOperand(i: 4).getReg();
621 Register MaskReg = MI.getOperand(i: 5).getReg();
622 bool IsSigned = BinOp == AtomicRMWInst::Min || BinOp == AtomicRMWInst::Max;
623 AtomicOrdering Ordering =
624 static_cast<AtomicOrdering>(MI.getOperand(i: IsSigned ? 7 : 6).getImm());
625
626 //
627 // .loophead:
628 // lr.w destreg, (alignedaddr)
629 // and scratch2, destreg, mask
630 // mv scratch1, destreg
631 // [sext scratch2 if signed min/max]
632 // ifnochangeneeded scratch2, incr, .looptail
633 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: getLRForRMW32(Ordering)), DestReg)
634 .addReg(RegNo: AddrReg);
635 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::AND), DestReg: Scratch2Reg)
636 .addReg(RegNo: DestReg)
637 .addReg(RegNo: MaskReg);
638 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::ADDI), DestReg: Scratch1Reg)
639 .addReg(RegNo: DestReg)
640 .addImm(Val: 0);
641
642 switch (BinOp) {
643 default:
644 llvm_unreachable("Unexpected AtomicRMW BinOp");
645 case AtomicRMWInst::Max: {
646 insertSext(DL, MBB: LoopHeadMBB, ValReg: Scratch2Reg, ShamtReg: MI.getOperand(i: 6).getReg());
647 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BGE))
648 .addReg(RegNo: Scratch2Reg)
649 .addReg(RegNo: IncrReg)
650 .addMBB(MBB: LoopTailMBB);
651 break;
652 }
653 case AtomicRMWInst::Min: {
654 insertSext(DL, MBB: LoopHeadMBB, ValReg: Scratch2Reg, ShamtReg: MI.getOperand(i: 6).getReg());
655 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BGE))
656 .addReg(RegNo: IncrReg)
657 .addReg(RegNo: Scratch2Reg)
658 .addMBB(MBB: LoopTailMBB);
659 break;
660 }
661 case AtomicRMWInst::UMax:
662 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BGEU))
663 .addReg(RegNo: Scratch2Reg)
664 .addReg(RegNo: IncrReg)
665 .addMBB(MBB: LoopTailMBB);
666 break;
667 case AtomicRMWInst::UMin:
668 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BGEU))
669 .addReg(RegNo: IncrReg)
670 .addReg(RegNo: Scratch2Reg)
671 .addMBB(MBB: LoopTailMBB);
672 break;
673 }
674
675 // .loopifbody:
676 // xor scratch1, destreg, incr
677 // and scratch1, scratch1, mask
678 // xor scratch1, destreg, scratch1
679 insertMaskedMerge(DL, MBB: LoopIfBodyMBB, DestReg: Scratch1Reg, OldValReg: DestReg, NewValReg: IncrReg, MaskReg,
680 ScratchReg: Scratch1Reg);
681
682 // .looptail:
683 // sc.w scratch1, scratch1, (addr)
684 // bnez scratch1, loop
685 BuildMI(BB: LoopTailMBB, MIMD: DL, MCID: TII->get(Opcode: getSCForRMW32(Ordering)), DestReg: Scratch1Reg)
686 .addReg(RegNo: Scratch1Reg)
687 .addReg(RegNo: AddrReg);
688 BuildMI(BB: LoopTailMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BNE))
689 .addReg(RegNo: Scratch1Reg)
690 .addReg(RegNo: RISCV::X0)
691 .addMBB(MBB: LoopHeadMBB);
692}
693
694bool RISCVExpandPseudoAtomicsImpl::expandAtomicMinMaxOp(
695 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
696 AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
697 MachineBasicBlock::iterator &NextMBBI) const {
698 // Using MIN(U)/MAX(U) is preferrable if permitted
699 if (STI->hasPermissiveZalrsc() && STI->hasStdExtZbb() && !IsMasked)
700 return expandAtomicBinOp(MBB, MBBI, BinOp, IsMasked, Width, NextMBBI);
701
702 MachineInstr &MI = *MBBI;
703 MachineFunction *MF = MBB.getParent();
704 auto LoopHeadMBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
705 auto LoopIfBodyMBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
706 auto LoopTailMBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
707 auto DoneMBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
708
709 // Insert new MBBs.
710 MF->insert(MBBI: ++MBB.getIterator(), MBB: LoopHeadMBB);
711 MF->insert(MBBI: ++LoopHeadMBB->getIterator(), MBB: LoopIfBodyMBB);
712 MF->insert(MBBI: ++LoopIfBodyMBB->getIterator(), MBB: LoopTailMBB);
713 MF->insert(MBBI: ++LoopTailMBB->getIterator(), MBB: DoneMBB);
714
715 // Set up successors and transfer remaining instructions to DoneMBB.
716 LoopHeadMBB->addSuccessor(Succ: LoopIfBodyMBB);
717 LoopHeadMBB->addSuccessor(Succ: LoopTailMBB);
718 LoopIfBodyMBB->addSuccessor(Succ: LoopTailMBB);
719 LoopTailMBB->addSuccessor(Succ: LoopHeadMBB);
720 LoopTailMBB->addSuccessor(Succ: DoneMBB);
721 DoneMBB->splice(Where: DoneMBB->end(), Other: &MBB, From: MI, To: MBB.end());
722 DoneMBB->transferSuccessors(FromMBB: &MBB);
723 MBB.addSuccessor(Succ: LoopHeadMBB);
724
725 if (!IsMasked)
726 doAtomicMinMaxOpExpansion(MI, LoopHeadMBB, LoopIfBodyMBB, LoopTailMBB,
727 BinOp, Width);
728 else
729 doMaskedAtomicMinMaxOpExpansion(MI, LoopHeadMBB, LoopIfBodyMBB, LoopTailMBB,
730 BinOp, Width);
731
732 NextMBBI = MBB.end();
733 MI.eraseFromParent();
734
735 LivePhysRegs LiveRegs;
736 computeAndAddLiveIns(LiveRegs, MBB&: *LoopHeadMBB);
737 computeAndAddLiveIns(LiveRegs, MBB&: *LoopIfBodyMBB);
738 computeAndAddLiveIns(LiveRegs, MBB&: *LoopTailMBB);
739 computeAndAddLiveIns(LiveRegs, MBB&: *DoneMBB);
740
741 return true;
742}
743
744// If a BNE on the cmpxchg comparison result immediately follows the cmpxchg
745// operation, it can be folded into the cmpxchg expansion by
746// modifying the branch within 'LoopHead' (which performs the same
747// comparison). This is a valid transformation because after altering the
748// LoopHead's BNE destination, the BNE following the cmpxchg becomes
749// redundant and and be deleted. In the case of a masked cmpxchg, an
750// appropriate AND and BNE must be matched.
751//
752// On success, returns true and deletes the matching BNE or AND+BNE, sets the
753// LoopHeadBNETarget argument to the target that should be used within the
754// loop head, and removes that block as a successor to MBB.
755bool RISCVExpandPseudoAtomicsImpl::tryToFoldBNEOnCmpXchgResult(
756 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg,
757 Register CmpValReg, Register MaskReg,
758 MachineBasicBlock *&LoopHeadBNETarget) const {
759 SmallVector<MachineInstr *> ToErase;
760 auto E = MBB.end();
761 if (MBBI == E)
762 return false;
763 MBBI = skipDebugInstructionsForward(It: MBBI, End: E);
764
765 // If we have a masked cmpxchg, match AND dst, DestReg, MaskReg.
766 if (MaskReg.isValid()) {
767 if (MBBI == E || MBBI->getOpcode() != RISCV::AND)
768 return false;
769 Register ANDOp1 = MBBI->getOperand(i: 1).getReg();
770 Register ANDOp2 = MBBI->getOperand(i: 2).getReg();
771 if (!(ANDOp1 == DestReg && ANDOp2 == MaskReg) &&
772 !(ANDOp1 == MaskReg && ANDOp2 == DestReg))
773 return false;
774 // We now expect the BNE to use the result of the AND as an operand.
775 DestReg = MBBI->getOperand(i: 0).getReg();
776 ToErase.push_back(Elt: &*MBBI);
777 MBBI = skipDebugInstructionsForward(It: std::next(x: MBBI), End: E);
778 }
779
780 // Match BNE DestReg, MaskReg.
781 if (MBBI == E || MBBI->getOpcode() != RISCV::BNE)
782 return false;
783 Register BNEOp0 = MBBI->getOperand(i: 0).getReg();
784 Register BNEOp1 = MBBI->getOperand(i: 1).getReg();
785 if (!(BNEOp0 == DestReg && BNEOp1 == CmpValReg) &&
786 !(BNEOp0 == CmpValReg && BNEOp1 == DestReg))
787 return false;
788
789 // Make sure the branch is the only user of the AND.
790 if (MaskReg.isValid()) {
791 if (BNEOp0 == DestReg && !MBBI->getOperand(i: 0).isKill())
792 return false;
793 if (BNEOp1 == DestReg && !MBBI->getOperand(i: 1).isKill())
794 return false;
795 }
796
797 ToErase.push_back(Elt: &*MBBI);
798 LoopHeadBNETarget = MBBI->getOperand(i: 2).getMBB();
799 MBBI = skipDebugInstructionsForward(It: std::next(x: MBBI), End: E);
800 if (MBBI != E)
801 return false;
802
803 MBB.removeSuccessor(Succ: LoopHeadBNETarget);
804 for (auto *MI : ToErase)
805 MI->eraseFromParent();
806 return true;
807}
808
809bool RISCVExpandPseudoAtomicsImpl::expandAtomicCmpXchg(
810 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool IsMasked,
811 int Width, MachineBasicBlock::iterator &NextMBBI) const {
812 MachineInstr &MI = *MBBI;
813 DebugLoc DL = MI.getDebugLoc();
814 MachineFunction *MF = MBB.getParent();
815 auto LoopHeadMBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
816 auto LoopTailMBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
817 auto DoneMBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
818
819 Register DestReg = MI.getOperand(i: 0).getReg();
820 Register ScratchReg = MI.getOperand(i: 1).getReg();
821 Register AddrReg = MI.getOperand(i: 2).getReg();
822 Register CmpValReg = MI.getOperand(i: 3).getReg();
823 Register NewValReg = MI.getOperand(i: 4).getReg();
824 Register MaskReg = IsMasked ? MI.getOperand(i: 5).getReg() : Register();
825
826 MachineBasicBlock *LoopHeadBNETarget = DoneMBB;
827 tryToFoldBNEOnCmpXchgResult(MBB, MBBI: std::next(x: MBBI), DestReg, CmpValReg, MaskReg,
828 LoopHeadBNETarget);
829
830 // Insert new MBBs.
831 MF->insert(MBBI: ++MBB.getIterator(), MBB: LoopHeadMBB);
832 MF->insert(MBBI: ++LoopHeadMBB->getIterator(), MBB: LoopTailMBB);
833 MF->insert(MBBI: ++LoopTailMBB->getIterator(), MBB: DoneMBB);
834
835 // Set up successors and transfer remaining instructions to DoneMBB.
836 LoopHeadMBB->addSuccessor(Succ: LoopTailMBB);
837 LoopHeadMBB->addSuccessor(Succ: LoopHeadBNETarget);
838 LoopTailMBB->addSuccessor(Succ: DoneMBB);
839 LoopTailMBB->addSuccessor(Succ: LoopHeadMBB);
840 DoneMBB->splice(Where: DoneMBB->end(), Other: &MBB, From: MI, To: MBB.end());
841 DoneMBB->transferSuccessors(FromMBB: &MBB);
842 MBB.addSuccessor(Succ: LoopHeadMBB);
843
844 AtomicOrdering Ordering =
845 static_cast<AtomicOrdering>(MI.getOperand(i: IsMasked ? 6 : 5).getImm());
846
847 if (!IsMasked) {
848 // .loophead:
849 // lr.[w|d] dest, (addr)
850 // bne dest, cmpval, done
851 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: getLRForRMW(Ordering, Width)), DestReg)
852 .addReg(RegNo: AddrReg);
853 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BNE))
854 .addReg(RegNo: DestReg)
855 .addReg(RegNo: CmpValReg)
856 .addMBB(MBB: LoopHeadBNETarget);
857 // .looptail:
858 // sc.[w|d] scratch, newval, (addr)
859 // bnez scratch, loophead
860 BuildMI(BB: LoopTailMBB, MIMD: DL, MCID: TII->get(Opcode: getSCForRMW(Ordering, Width)), DestReg: ScratchReg)
861 .addReg(RegNo: NewValReg)
862 .addReg(RegNo: AddrReg);
863 BuildMI(BB: LoopTailMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BNE))
864 .addReg(RegNo: ScratchReg)
865 .addReg(RegNo: RISCV::X0)
866 .addMBB(MBB: LoopHeadMBB);
867 } else {
868 // .loophead:
869 // lr.w dest, (addr)
870 // and scratch, dest, mask
871 // bne scratch, cmpval, done
872 Register MaskReg = MI.getOperand(i: 5).getReg();
873 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: getLRForRMW(Ordering, Width)), DestReg)
874 .addReg(RegNo: AddrReg);
875 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::AND), DestReg: ScratchReg)
876 .addReg(RegNo: DestReg)
877 .addReg(RegNo: MaskReg);
878 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BNE))
879 .addReg(RegNo: ScratchReg)
880 .addReg(RegNo: CmpValReg)
881 .addMBB(MBB: LoopHeadBNETarget);
882
883 // .looptail:
884 // xor scratch, dest, newval
885 // and scratch, scratch, mask
886 // xor scratch, dest, scratch
887 // sc.w scratch, scratch, (adrr)
888 // bnez scratch, loophead
889 // A zero newval needs only the last xor: scratch still holds dest & mask
890 // from .loophead.
891 if (NewValReg == RISCV::X0)
892 BuildMI(BB: LoopTailMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::XOR), DestReg: ScratchReg)
893 .addReg(RegNo: DestReg)
894 .addReg(RegNo: ScratchReg);
895 else
896 insertMaskedMerge(DL, MBB: LoopTailMBB, DestReg: ScratchReg, OldValReg: DestReg, NewValReg,
897 MaskReg, ScratchReg);
898 BuildMI(BB: LoopTailMBB, MIMD: DL, MCID: TII->get(Opcode: getSCForRMW(Ordering, Width)), DestReg: ScratchReg)
899 .addReg(RegNo: ScratchReg)
900 .addReg(RegNo: AddrReg);
901 BuildMI(BB: LoopTailMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BNE))
902 .addReg(RegNo: ScratchReg)
903 .addReg(RegNo: RISCV::X0)
904 .addMBB(MBB: LoopHeadMBB);
905 }
906
907 NextMBBI = MBB.end();
908 MI.eraseFromParent();
909
910 LivePhysRegs LiveRegs;
911 computeAndAddLiveIns(LiveRegs, MBB&: *LoopHeadMBB);
912 computeAndAddLiveIns(LiveRegs, MBB&: *LoopTailMBB);
913 computeAndAddLiveIns(LiveRegs, MBB&: *DoneMBB);
914
915 return true;
916}
917
918char RISCVExpandPseudoAtomicsLegacy::ID = 0;
919
920INITIALIZE_PASS(RISCVExpandPseudoAtomicsLegacy, "riscv-expand-pseudo-atomics",
921 RISCV_EXPAND_PSEUDO_ATOMICS_NAME, false, false)
922
923FunctionPass *llvm::createRISCVExpandPseudoAtomicsLegacyPass() {
924 return new RISCVExpandPseudoAtomicsLegacy();
925}
926
927PreservedAnalyses
928RISCVExpandPseudoAtomicsPass::run(MachineFunction &MF,
929 MachineFunctionAnalysisManager &MFAM) {
930 bool Changed = RISCVExpandPseudoAtomicsImpl().run(MF);
931 if (!Changed)
932 return PreservedAnalyses::all();
933 return getMachineFunctionPassPreservedAnalyses();
934}
935