1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains one of the several passes that expand pseudo instructions
10// into target instructions. This pass is run at the last possible moment,
11// avoiding the possibility for other passes to break the requirements for
12// forward progress in the LR/SC block.
13//
14//===----------------------------------------------------------------------===//
15
16#include "RISCV.h"
17#include "RISCVExpandPseudoBase.h"
18#include "RISCVInstrInfo.h"
19#include "RISCVSubtarget.h"
20#include "llvm/CodeGen/LivePhysRegs.h"
21#include "llvm/CodeGen/MachineFunctionPass.h"
22#include "llvm/CodeGen/MachineInstrBuilder.h"
23
24using namespace llvm;
25
26#define RISCV_EXPAND_PSEUDO_ATOMICS_NAME \
27 "RISC-V Pseudo Instruction Expansion - Atomics"
28
29namespace {
30
31class RISCVExpandPseudoAtomicsImpl final : public RISCVExpandPseudoImplBase {
32 bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
33 MachineBasicBlock::iterator &NextMBBI) const override;
34
35 bool expandAtomicBinOp(MachineBasicBlock &MBB,
36 MachineBasicBlock::iterator MBBI, AtomicRMWInst::BinOp,
37 bool IsMasked, int Width,
38 MachineBasicBlock::iterator &NextMBBI) const;
39
40 bool expandAtomicMinMaxOp(MachineBasicBlock &MBB,
41 MachineBasicBlock::iterator MBBI,
42 AtomicRMWInst::BinOp, bool IsMasked, int Width,
43 MachineBasicBlock::iterator &NextMBBI) const;
44
45 bool expandAtomicCmpXchg(MachineBasicBlock &MBB,
46 MachineBasicBlock::iterator MBBI, bool IsMasked,
47 int Width,
48 MachineBasicBlock::iterator &NextMBBI) const;
49
50 unsigned getLRForRMW32(AtomicOrdering Ordering) const;
51 unsigned getSCForRMW32(AtomicOrdering Ordering) const;
52 unsigned getLRForRMW64(AtomicOrdering Ordering) const;
53 unsigned getSCForRMW64(AtomicOrdering Ordering) const;
54 unsigned getLRForRMW(AtomicOrdering Ordering, int Width) const;
55 unsigned getSCForRMW(AtomicOrdering Ordering, int Width) const;
56
57 void doAtomicBinOpExpansion(MachineInstr &MI, MachineBasicBlock *LoopMBB,
58 AtomicRMWInst::BinOp BinOp, int Width) const;
59 void insertMaskedMerge(DebugLoc DL, MachineBasicBlock *MBB, Register DestReg,
60 Register OldValReg, Register NewValReg,
61 Register MaskReg, Register ScratchReg) const;
62 void doMaskedAtomicBinOpExpansion(MachineInstr &MI,
63 MachineBasicBlock *LoopMBB,
64 AtomicRMWInst::BinOp BinOp,
65 int Width) const;
66 void insertSext(DebugLoc DL, MachineBasicBlock *MBB, Register ValReg,
67 Register ShamtReg) const;
68 void doAtomicMinMaxOpExpansion(MachineInstr &MI,
69 MachineBasicBlock *LoopHeadMBB,
70 MachineBasicBlock *LoopIfBodyMBB,
71 MachineBasicBlock *LoopTailMBB,
72 AtomicRMWInst::BinOp BinOp, int Width) const;
73 void doMaskedAtomicMinMaxOpExpansion(MachineInstr &MI,
74 MachineBasicBlock *LoopHeadMBB,
75 MachineBasicBlock *LoopIfBodyMBB,
76 MachineBasicBlock *LoopTailMBB,
77 AtomicRMWInst::BinOp BinOp,
78 int Width) const;
79 bool tryToFoldBNEOnCmpXchgResult(MachineBasicBlock &MBB,
80 MachineBasicBlock::iterator MBBI,
81 Register DestReg, Register CmpValReg,
82 Register MaskReg,
83 MachineBasicBlock *&LoopHeadBNETarget) const;
84};
85
86class RISCVExpandPseudoAtomicsLegacy : public MachineFunctionPass {
87public:
88 static char ID;
89
90 RISCVExpandPseudoAtomicsLegacy() : MachineFunctionPass(ID) {}
91
92 bool runOnMachineFunction(MachineFunction &MF) override {
93 return RISCVExpandPseudoAtomicsImpl().run(MF);
94 }
95
96 StringRef getPassName() const override {
97 return RISCV_EXPAND_PSEUDO_ATOMICS_NAME;
98 }
99};
100
101} // anonymous namespace
102
103bool RISCVExpandPseudoAtomicsImpl::expandMI(
104 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
105 MachineBasicBlock::iterator &NextMBBI) const {
106 // RISCVInstrInfo::getInstSizeInBytes expects that the total size of the
107 // expanded instructions for each pseudo is correct in the Size field of the
108 // tablegen definition for the pseudo.
109 switch (MBBI->getOpcode()) {
110 case RISCV::PseudoAtomicSwap32:
111 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, IsMasked: false, Width: 32,
112 NextMBBI);
113 case RISCV::PseudoAtomicSwap64:
114 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, IsMasked: false, Width: 64,
115 NextMBBI);
116 case RISCV::PseudoAtomicLoadAdd32:
117 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, IsMasked: false, Width: 32,
118 NextMBBI);
119 case RISCV::PseudoAtomicLoadAdd64:
120 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, IsMasked: false, Width: 64,
121 NextMBBI);
122 case RISCV::PseudoAtomicLoadSub32:
123 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, IsMasked: false, Width: 32,
124 NextMBBI);
125 case RISCV::PseudoAtomicLoadSub64:
126 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, IsMasked: false, Width: 64,
127 NextMBBI);
128 case RISCV::PseudoAtomicLoadAnd32:
129 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::And, IsMasked: false, Width: 32,
130 NextMBBI);
131 case RISCV::PseudoAtomicLoadAnd64:
132 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::And, IsMasked: false, Width: 64,
133 NextMBBI);
134 case RISCV::PseudoAtomicLoadOr32:
135 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Or, IsMasked: false, Width: 32, NextMBBI);
136 case RISCV::PseudoAtomicLoadOr64:
137 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Or, IsMasked: false, Width: 64, NextMBBI);
138 case RISCV::PseudoAtomicLoadXor32:
139 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xor, IsMasked: false, Width: 32,
140 NextMBBI);
141 case RISCV::PseudoAtomicLoadXor64:
142 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xor, IsMasked: false, Width: 64,
143 NextMBBI);
144 case RISCV::PseudoAtomicLoadNand32:
145 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, IsMasked: false, Width: 32,
146 NextMBBI);
147 case RISCV::PseudoAtomicLoadNand64:
148 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, IsMasked: false, Width: 64,
149 NextMBBI);
150 case RISCV::PseudoAtomicLoadMin32:
151 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Min, IsMasked: false, Width: 32,
152 NextMBBI);
153 case RISCV::PseudoAtomicLoadMin64:
154 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Min, IsMasked: false, Width: 64,
155 NextMBBI);
156 case RISCV::PseudoAtomicLoadMax32:
157 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, IsMasked: false, Width: 32,
158 NextMBBI);
159 case RISCV::PseudoAtomicLoadMax64:
160 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, IsMasked: false, Width: 64,
161 NextMBBI);
162 case RISCV::PseudoAtomicLoadUMin32:
163 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, IsMasked: false, Width: 32,
164 NextMBBI);
165 case RISCV::PseudoAtomicLoadUMin64:
166 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, IsMasked: false, Width: 64,
167 NextMBBI);
168 case RISCV::PseudoAtomicLoadUMax32:
169 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, IsMasked: false, Width: 32,
170 NextMBBI);
171 case RISCV::PseudoAtomicLoadUMax64:
172 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, IsMasked: false, Width: 64,
173 NextMBBI);
174 case RISCV::PseudoMaskedAtomicSwap32:
175 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, IsMasked: true, Width: 32,
176 NextMBBI);
177 case RISCV::PseudoMaskedAtomicLoadAdd32:
178 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Add, IsMasked: true, Width: 32, NextMBBI);
179 case RISCV::PseudoMaskedAtomicLoadSub32:
180 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Sub, IsMasked: true, Width: 32, NextMBBI);
181 case RISCV::PseudoMaskedAtomicLoadNand32:
182 return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, IsMasked: true, Width: 32,
183 NextMBBI);
184 case RISCV::PseudoMaskedAtomicLoadMax32:
185 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Max, IsMasked: true, Width: 32,
186 NextMBBI);
187 case RISCV::PseudoMaskedAtomicLoadMin32:
188 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::Min, IsMasked: true, Width: 32,
189 NextMBBI);
190 case RISCV::PseudoMaskedAtomicLoadUMax32:
191 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMax, IsMasked: true, Width: 32,
192 NextMBBI);
193 case RISCV::PseudoMaskedAtomicLoadUMin32:
194 return expandAtomicMinMaxOp(MBB, MBBI, AtomicRMWInst::UMin, IsMasked: true, Width: 32,
195 NextMBBI);
196 case RISCV::PseudoCmpXchg32:
197 return expandAtomicCmpXchg(MBB, MBBI, IsMasked: false, Width: 32, NextMBBI);
198 case RISCV::PseudoCmpXchg64:
199 return expandAtomicCmpXchg(MBB, MBBI, IsMasked: false, Width: 64, NextMBBI);
200 case RISCV::PseudoMaskedCmpXchg32:
201 return expandAtomicCmpXchg(MBB, MBBI, IsMasked: true, Width: 32, NextMBBI);
202 }
203
204 return false;
205}
206
207unsigned
208RISCVExpandPseudoAtomicsImpl::getLRForRMW32(AtomicOrdering Ordering) const {
209 switch (Ordering) {
210 default:
211 llvm_unreachable("Unexpected AtomicOrdering");
212 case AtomicOrdering::Monotonic:
213 return RISCV::LR_W;
214 case AtomicOrdering::Acquire:
215 if (STI->hasStdExtZtso())
216 return RISCV::LR_W;
217 return RISCV::LR_W_AQ;
218 case AtomicOrdering::Release:
219 return RISCV::LR_W;
220 case AtomicOrdering::AcquireRelease:
221 if (STI->hasStdExtZtso())
222 return RISCV::LR_W;
223 return RISCV::LR_W_AQ;
224 case AtomicOrdering::SequentiallyConsistent:
225 return RISCV::LR_W_AQRL;
226 }
227}
228
229unsigned
230RISCVExpandPseudoAtomicsImpl::getSCForRMW32(AtomicOrdering Ordering) const {
231 switch (Ordering) {
232 default:
233 llvm_unreachable("Unexpected AtomicOrdering");
234 case AtomicOrdering::Monotonic:
235 return RISCV::SC_W;
236 case AtomicOrdering::Acquire:
237 return RISCV::SC_W;
238 case AtomicOrdering::Release:
239 if (STI->hasStdExtZtso())
240 return RISCV::SC_W;
241 return RISCV::SC_W_RL;
242 case AtomicOrdering::AcquireRelease:
243 if (STI->hasStdExtZtso())
244 return RISCV::SC_W;
245 return RISCV::SC_W_RL;
246 case AtomicOrdering::SequentiallyConsistent:
247 return RISCV::SC_W_RL;
248 }
249}
250
251unsigned
252RISCVExpandPseudoAtomicsImpl::getLRForRMW64(AtomicOrdering Ordering) const {
253 switch (Ordering) {
254 default:
255 llvm_unreachable("Unexpected AtomicOrdering");
256 case AtomicOrdering::Monotonic:
257 return RISCV::LR_D;
258 case AtomicOrdering::Acquire:
259 if (STI->hasStdExtZtso())
260 return RISCV::LR_D;
261 return RISCV::LR_D_AQ;
262 case AtomicOrdering::Release:
263 return RISCV::LR_D;
264 case AtomicOrdering::AcquireRelease:
265 if (STI->hasStdExtZtso())
266 return RISCV::LR_D;
267 return RISCV::LR_D_AQ;
268 case AtomicOrdering::SequentiallyConsistent:
269 return RISCV::LR_D_AQRL;
270 }
271}
272
273unsigned
274RISCVExpandPseudoAtomicsImpl::getSCForRMW64(AtomicOrdering Ordering) const {
275 switch (Ordering) {
276 default:
277 llvm_unreachable("Unexpected AtomicOrdering");
278 case AtomicOrdering::Monotonic:
279 return RISCV::SC_D;
280 case AtomicOrdering::Acquire:
281 return RISCV::SC_D;
282 case AtomicOrdering::Release:
283 if (STI->hasStdExtZtso())
284 return RISCV::SC_D;
285 return RISCV::SC_D_RL;
286 case AtomicOrdering::AcquireRelease:
287 if (STI->hasStdExtZtso())
288 return RISCV::SC_D;
289 return RISCV::SC_D_RL;
290 case AtomicOrdering::SequentiallyConsistent:
291 return RISCV::SC_D_RL;
292 }
293}
294
295unsigned RISCVExpandPseudoAtomicsImpl::getLRForRMW(AtomicOrdering Ordering,
296 int Width) const {
297 if (Width == 32)
298 return getLRForRMW32(Ordering);
299 if (Width == 64)
300 return getLRForRMW64(Ordering);
301 llvm_unreachable("Unexpected LR width\n");
302}
303
304unsigned RISCVExpandPseudoAtomicsImpl::getSCForRMW(AtomicOrdering Ordering,
305 int Width) const {
306 if (Width == 32)
307 return getSCForRMW32(Ordering);
308 if (Width == 64)
309 return getSCForRMW64(Ordering);
310 llvm_unreachable("Unexpected SC width\n");
311}
312
313void RISCVExpandPseudoAtomicsImpl::doAtomicBinOpExpansion(
314 MachineInstr &MI, MachineBasicBlock *LoopMBB, AtomicRMWInst::BinOp BinOp,
315 int Width) const {
316 DebugLoc DL = MI.getDebugLoc();
317 Register DestReg = MI.getOperand(i: 0).getReg();
318 Register ScratchReg = MI.getOperand(i: 1).getReg();
319 Register AddrReg = MI.getOperand(i: 2).getReg();
320 Register IncrReg = MI.getOperand(i: 3).getReg();
321 AtomicOrdering Ordering =
322 static_cast<AtomicOrdering>(MI.getOperand(i: 4).getImm());
323
324 // .loop:
325 // lr.[w|d] dest, (addr)
326 // binop scratch, dest, val
327 // sc.[w|d] scratch, scratch, (addr)
328 // bnez scratch, loop
329 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: getLRForRMW(Ordering, Width)), DestReg)
330 .addReg(RegNo: AddrReg);
331 switch (BinOp) {
332 default:
333 llvm_unreachable("Unexpected AtomicRMW BinOp");
334 case AtomicRMWInst::Xchg:
335 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::ADDI), DestReg: ScratchReg)
336 .addReg(RegNo: IncrReg)
337 .addImm(Val: 0);
338 break;
339 case AtomicRMWInst::Add:
340 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::ADD), DestReg: ScratchReg)
341 .addReg(RegNo: DestReg)
342 .addReg(RegNo: IncrReg);
343 break;
344 case AtomicRMWInst::Sub:
345 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::SUB), DestReg: ScratchReg)
346 .addReg(RegNo: DestReg)
347 .addReg(RegNo: IncrReg);
348 break;
349 case AtomicRMWInst::And:
350 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::AND), DestReg: ScratchReg)
351 .addReg(RegNo: DestReg)
352 .addReg(RegNo: IncrReg);
353 break;
354 case AtomicRMWInst::Or:
355 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::OR), DestReg: ScratchReg)
356 .addReg(RegNo: DestReg)
357 .addReg(RegNo: IncrReg);
358 break;
359 case AtomicRMWInst::Xor:
360 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::XOR), DestReg: ScratchReg)
361 .addReg(RegNo: DestReg)
362 .addReg(RegNo: IncrReg);
363 break;
364 case AtomicRMWInst::Nand:
365 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::AND), DestReg: ScratchReg)
366 .addReg(RegNo: DestReg)
367 .addReg(RegNo: IncrReg);
368 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::XORI), DestReg: ScratchReg)
369 .addReg(RegNo: ScratchReg)
370 .addImm(Val: -1);
371 break;
372 case AtomicRMWInst::Max:
373 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::MAX), DestReg: ScratchReg)
374 .addReg(RegNo: DestReg)
375 .addReg(RegNo: IncrReg);
376 break;
377 case AtomicRMWInst::Min:
378 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::MIN), DestReg: ScratchReg)
379 .addReg(RegNo: DestReg)
380 .addReg(RegNo: IncrReg);
381 break;
382 case AtomicRMWInst::UMax:
383 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::MAXU), DestReg: ScratchReg)
384 .addReg(RegNo: DestReg)
385 .addReg(RegNo: IncrReg);
386 break;
387 case AtomicRMWInst::UMin:
388 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::MINU), DestReg: ScratchReg)
389 .addReg(RegNo: DestReg)
390 .addReg(RegNo: IncrReg);
391 break;
392 }
393 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: getSCForRMW(Ordering, Width)), DestReg: ScratchReg)
394 .addReg(RegNo: ScratchReg)
395 .addReg(RegNo: AddrReg);
396 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BNE))
397 .addReg(RegNo: ScratchReg)
398 .addReg(RegNo: RISCV::X0)
399 .addMBB(MBB: LoopMBB);
400}
401
402void RISCVExpandPseudoAtomicsImpl::insertMaskedMerge(
403 DebugLoc DL, MachineBasicBlock *MBB, Register DestReg, Register OldValReg,
404 Register NewValReg, Register MaskReg, Register ScratchReg) const {
405 assert(OldValReg != ScratchReg && "OldValReg and ScratchReg must be unique");
406 assert(OldValReg != MaskReg && "OldValReg and MaskReg must be unique");
407 assert(ScratchReg != MaskReg && "ScratchReg and MaskReg must be unique");
408
409 // We select bits from newval and oldval using:
410 // https://graphics.stanford.edu/~seander/bithacks.html#MaskedMerge
411 // r = oldval ^ ((oldval ^ newval) & masktargetdata);
412 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::XOR), DestReg: ScratchReg)
413 .addReg(RegNo: OldValReg)
414 .addReg(RegNo: NewValReg);
415 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::AND), DestReg: ScratchReg)
416 .addReg(RegNo: ScratchReg)
417 .addReg(RegNo: MaskReg);
418 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::XOR), DestReg)
419 .addReg(RegNo: OldValReg)
420 .addReg(RegNo: ScratchReg);
421}
422
423void RISCVExpandPseudoAtomicsImpl::doMaskedAtomicBinOpExpansion(
424 MachineInstr &MI, MachineBasicBlock *LoopMBB, AtomicRMWInst::BinOp BinOp,
425 int Width) const {
426 DebugLoc DL = MI.getDebugLoc();
427 assert(Width == 32 && "Should never need to expand masked 64-bit operations");
428 Register DestReg = MI.getOperand(i: 0).getReg();
429 Register ScratchReg = MI.getOperand(i: 1).getReg();
430 Register AddrReg = MI.getOperand(i: 2).getReg();
431 Register IncrReg = MI.getOperand(i: 3).getReg();
432 Register MaskReg = MI.getOperand(i: 4).getReg();
433 AtomicOrdering Ordering =
434 static_cast<AtomicOrdering>(MI.getOperand(i: 5).getImm());
435
436 // .loop:
437 // lr.w destreg, (alignedaddr)
438 // binop scratch, destreg, incr
439 // xor scratch, destreg, scratch
440 // and scratch, scratch, masktargetdata
441 // xor scratch, destreg, scratch
442 // sc.w scratch, scratch, (alignedaddr)
443 // bnez scratch, loop
444 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: getLRForRMW32(Ordering)), DestReg)
445 .addReg(RegNo: AddrReg);
446 switch (BinOp) {
447 default:
448 llvm_unreachable("Unexpected AtomicRMW BinOp");
449 case AtomicRMWInst::Xchg:
450 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::ADDI), DestReg: ScratchReg)
451 .addReg(RegNo: IncrReg)
452 .addImm(Val: 0);
453 break;
454 case AtomicRMWInst::Add:
455 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::ADD), DestReg: ScratchReg)
456 .addReg(RegNo: DestReg)
457 .addReg(RegNo: IncrReg);
458 break;
459 case AtomicRMWInst::Sub:
460 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::SUB), DestReg: ScratchReg)
461 .addReg(RegNo: DestReg)
462 .addReg(RegNo: IncrReg);
463 break;
464 case AtomicRMWInst::Nand:
465 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::AND), DestReg: ScratchReg)
466 .addReg(RegNo: DestReg)
467 .addReg(RegNo: IncrReg);
468 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::XORI), DestReg: ScratchReg)
469 .addReg(RegNo: ScratchReg)
470 .addImm(Val: -1);
471 break;
472 }
473
474 insertMaskedMerge(DL, MBB: LoopMBB, DestReg: ScratchReg, OldValReg: DestReg, NewValReg: ScratchReg, MaskReg,
475 ScratchReg);
476
477 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: getSCForRMW32(Ordering)), DestReg: ScratchReg)
478 .addReg(RegNo: ScratchReg)
479 .addReg(RegNo: AddrReg);
480 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BNE))
481 .addReg(RegNo: ScratchReg)
482 .addReg(RegNo: RISCV::X0)
483 .addMBB(MBB: LoopMBB);
484}
485
486bool RISCVExpandPseudoAtomicsImpl::expandAtomicBinOp(
487 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
488 AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
489 MachineBasicBlock::iterator &NextMBBI) const {
490 MachineInstr &MI = *MBBI;
491 MachineFunction *MF = MBB.getParent();
492 auto LoopMBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
493 auto DoneMBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
494
495 // Insert new MBBs.
496 MF->insert(MBBI: ++MBB.getIterator(), MBB: LoopMBB);
497 MF->insert(MBBI: ++LoopMBB->getIterator(), MBB: DoneMBB);
498
499 // Set up successors and transfer remaining instructions to DoneMBB.
500 LoopMBB->addSuccessor(Succ: LoopMBB);
501 LoopMBB->addSuccessor(Succ: DoneMBB);
502 DoneMBB->splice(Where: DoneMBB->end(), Other: &MBB, From: MI, To: MBB.end());
503 DoneMBB->transferSuccessors(FromMBB: &MBB);
504 MBB.addSuccessor(Succ: LoopMBB);
505
506 if (!IsMasked)
507 doAtomicBinOpExpansion(MI, LoopMBB, BinOp, Width);
508 else
509 doMaskedAtomicBinOpExpansion(MI, LoopMBB, BinOp, Width);
510
511 NextMBBI = MBB.end();
512 MI.eraseFromParent();
513
514 LivePhysRegs LiveRegs;
515 computeAndAddLiveIns(LiveRegs, MBB&: *LoopMBB);
516 computeAndAddLiveIns(LiveRegs, MBB&: *DoneMBB);
517
518 return true;
519}
520
521void RISCVExpandPseudoAtomicsImpl::insertSext(DebugLoc DL,
522 MachineBasicBlock *MBB,
523 Register ValReg,
524 Register ShamtReg) const {
525 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::SLL), DestReg: ValReg)
526 .addReg(RegNo: ValReg)
527 .addReg(RegNo: ShamtReg);
528 BuildMI(BB: MBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::SRA), DestReg: ValReg)
529 .addReg(RegNo: ValReg)
530 .addReg(RegNo: ShamtReg);
531}
532
533void RISCVExpandPseudoAtomicsImpl::doAtomicMinMaxOpExpansion(
534 MachineInstr &MI, MachineBasicBlock *LoopHeadMBB,
535 MachineBasicBlock *LoopIfBodyMBB, MachineBasicBlock *LoopTailMBB,
536 AtomicRMWInst::BinOp BinOp, int Width) const {
537 DebugLoc DL = MI.getDebugLoc();
538 Register DestReg = MI.getOperand(i: 0).getReg();
539 Register ScratchReg = MI.getOperand(i: 1).getReg();
540 Register AddrReg = MI.getOperand(i: 2).getReg();
541 Register IncrReg = MI.getOperand(i: 3).getReg();
542 AtomicOrdering Ordering =
543 static_cast<AtomicOrdering>(MI.getOperand(i: 4).getImm());
544
545 // .loophead:
546 // lr.[w|d] dest, (addr)
547 // mv scratch, dest
548 // ifnochangeneeded scratch, incr, .looptail
549 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: getLRForRMW(Ordering, Width)), DestReg)
550 .addReg(RegNo: AddrReg);
551 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::ADDI), DestReg: ScratchReg)
552 .addReg(RegNo: DestReg)
553 .addImm(Val: 0);
554 switch (BinOp) {
555 default:
556 llvm_unreachable("Unexpected AtomicRMW BinOp");
557 case AtomicRMWInst::Max: {
558 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BGE))
559 .addReg(RegNo: ScratchReg)
560 .addReg(RegNo: IncrReg)
561 .addMBB(MBB: LoopTailMBB);
562 break;
563 }
564 case AtomicRMWInst::Min: {
565 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BGE))
566 .addReg(RegNo: IncrReg)
567 .addReg(RegNo: ScratchReg)
568 .addMBB(MBB: LoopTailMBB);
569 break;
570 }
571 case AtomicRMWInst::UMax:
572 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BGEU))
573 .addReg(RegNo: ScratchReg)
574 .addReg(RegNo: IncrReg)
575 .addMBB(MBB: LoopTailMBB);
576 break;
577 case AtomicRMWInst::UMin:
578 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BGEU))
579 .addReg(RegNo: IncrReg)
580 .addReg(RegNo: ScratchReg)
581 .addMBB(MBB: LoopTailMBB);
582 break;
583 }
584
585 // .loopifbody:
586 // mv scratch, incr
587 BuildMI(BB: LoopIfBodyMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::ADDI), DestReg: ScratchReg)
588 .addReg(RegNo: IncrReg)
589 .addImm(Val: 0);
590
591 // .looptail:
592 // sc.[w|d] scratch, scratch, (addr)
593 // bnez scratch, loop
594 BuildMI(BB: LoopTailMBB, MIMD: DL, MCID: TII->get(Opcode: getSCForRMW(Ordering, Width)), DestReg: ScratchReg)
595 .addReg(RegNo: ScratchReg)
596 .addReg(RegNo: AddrReg);
597 BuildMI(BB: LoopTailMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BNE))
598 .addReg(RegNo: ScratchReg)
599 .addReg(RegNo: RISCV::X0)
600 .addMBB(MBB: LoopHeadMBB);
601}
602
603void RISCVExpandPseudoAtomicsImpl::doMaskedAtomicMinMaxOpExpansion(
604 MachineInstr &MI, MachineBasicBlock *LoopHeadMBB,
605 MachineBasicBlock *LoopIfBodyMBB, MachineBasicBlock *LoopTailMBB,
606 AtomicRMWInst::BinOp BinOp, int Width) const {
607 DebugLoc DL = MI.getDebugLoc();
608 assert(Width == 32 && "Should never need to expand masked 64-bit operations");
609 Register DestReg = MI.getOperand(i: 0).getReg();
610 Register Scratch1Reg = MI.getOperand(i: 1).getReg();
611 Register Scratch2Reg = MI.getOperand(i: 2).getReg();
612 Register AddrReg = MI.getOperand(i: 3).getReg();
613 Register IncrReg = MI.getOperand(i: 4).getReg();
614 Register MaskReg = MI.getOperand(i: 5).getReg();
615 bool IsSigned = BinOp == AtomicRMWInst::Min || BinOp == AtomicRMWInst::Max;
616 AtomicOrdering Ordering =
617 static_cast<AtomicOrdering>(MI.getOperand(i: IsSigned ? 7 : 6).getImm());
618
619 //
620 // .loophead:
621 // lr.w destreg, (alignedaddr)
622 // and scratch2, destreg, mask
623 // mv scratch1, destreg
624 // [sext scratch2 if signed min/max]
625 // ifnochangeneeded scratch2, incr, .looptail
626 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: getLRForRMW32(Ordering)), DestReg)
627 .addReg(RegNo: AddrReg);
628 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::AND), DestReg: Scratch2Reg)
629 .addReg(RegNo: DestReg)
630 .addReg(RegNo: MaskReg);
631 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::ADDI), DestReg: Scratch1Reg)
632 .addReg(RegNo: DestReg)
633 .addImm(Val: 0);
634
635 switch (BinOp) {
636 default:
637 llvm_unreachable("Unexpected AtomicRMW BinOp");
638 case AtomicRMWInst::Max: {
639 insertSext(DL, MBB: LoopHeadMBB, ValReg: Scratch2Reg, ShamtReg: MI.getOperand(i: 6).getReg());
640 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BGE))
641 .addReg(RegNo: Scratch2Reg)
642 .addReg(RegNo: IncrReg)
643 .addMBB(MBB: LoopTailMBB);
644 break;
645 }
646 case AtomicRMWInst::Min: {
647 insertSext(DL, MBB: LoopHeadMBB, ValReg: Scratch2Reg, ShamtReg: MI.getOperand(i: 6).getReg());
648 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BGE))
649 .addReg(RegNo: IncrReg)
650 .addReg(RegNo: Scratch2Reg)
651 .addMBB(MBB: LoopTailMBB);
652 break;
653 }
654 case AtomicRMWInst::UMax:
655 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BGEU))
656 .addReg(RegNo: Scratch2Reg)
657 .addReg(RegNo: IncrReg)
658 .addMBB(MBB: LoopTailMBB);
659 break;
660 case AtomicRMWInst::UMin:
661 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BGEU))
662 .addReg(RegNo: IncrReg)
663 .addReg(RegNo: Scratch2Reg)
664 .addMBB(MBB: LoopTailMBB);
665 break;
666 }
667
668 // .loopifbody:
669 // xor scratch1, destreg, incr
670 // and scratch1, scratch1, mask
671 // xor scratch1, destreg, scratch1
672 insertMaskedMerge(DL, MBB: LoopIfBodyMBB, DestReg: Scratch1Reg, OldValReg: DestReg, NewValReg: IncrReg, MaskReg,
673 ScratchReg: Scratch1Reg);
674
675 // .looptail:
676 // sc.w scratch1, scratch1, (addr)
677 // bnez scratch1, loop
678 BuildMI(BB: LoopTailMBB, MIMD: DL, MCID: TII->get(Opcode: getSCForRMW32(Ordering)), DestReg: Scratch1Reg)
679 .addReg(RegNo: Scratch1Reg)
680 .addReg(RegNo: AddrReg);
681 BuildMI(BB: LoopTailMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BNE))
682 .addReg(RegNo: Scratch1Reg)
683 .addReg(RegNo: RISCV::X0)
684 .addMBB(MBB: LoopHeadMBB);
685}
686
687bool RISCVExpandPseudoAtomicsImpl::expandAtomicMinMaxOp(
688 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
689 AtomicRMWInst::BinOp BinOp, bool IsMasked, int Width,
690 MachineBasicBlock::iterator &NextMBBI) const {
691 // Using MIN(U)/MAX(U) is preferrable if permitted
692 if (STI->hasPermissiveZalrsc() && STI->hasStdExtZbb() && !IsMasked)
693 return expandAtomicBinOp(MBB, MBBI, BinOp, IsMasked, Width, NextMBBI);
694
695 MachineInstr &MI = *MBBI;
696 MachineFunction *MF = MBB.getParent();
697 auto LoopHeadMBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
698 auto LoopIfBodyMBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
699 auto LoopTailMBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
700 auto DoneMBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
701
702 // Insert new MBBs.
703 MF->insert(MBBI: ++MBB.getIterator(), MBB: LoopHeadMBB);
704 MF->insert(MBBI: ++LoopHeadMBB->getIterator(), MBB: LoopIfBodyMBB);
705 MF->insert(MBBI: ++LoopIfBodyMBB->getIterator(), MBB: LoopTailMBB);
706 MF->insert(MBBI: ++LoopTailMBB->getIterator(), MBB: DoneMBB);
707
708 // Set up successors and transfer remaining instructions to DoneMBB.
709 LoopHeadMBB->addSuccessor(Succ: LoopIfBodyMBB);
710 LoopHeadMBB->addSuccessor(Succ: LoopTailMBB);
711 LoopIfBodyMBB->addSuccessor(Succ: LoopTailMBB);
712 LoopTailMBB->addSuccessor(Succ: LoopHeadMBB);
713 LoopTailMBB->addSuccessor(Succ: DoneMBB);
714 DoneMBB->splice(Where: DoneMBB->end(), Other: &MBB, From: MI, To: MBB.end());
715 DoneMBB->transferSuccessors(FromMBB: &MBB);
716 MBB.addSuccessor(Succ: LoopHeadMBB);
717
718 if (!IsMasked)
719 doAtomicMinMaxOpExpansion(MI, LoopHeadMBB, LoopIfBodyMBB, LoopTailMBB,
720 BinOp, Width);
721 else
722 doMaskedAtomicMinMaxOpExpansion(MI, LoopHeadMBB, LoopIfBodyMBB, LoopTailMBB,
723 BinOp, Width);
724
725 NextMBBI = MBB.end();
726 MI.eraseFromParent();
727
728 LivePhysRegs LiveRegs;
729 computeAndAddLiveIns(LiveRegs, MBB&: *LoopHeadMBB);
730 computeAndAddLiveIns(LiveRegs, MBB&: *LoopIfBodyMBB);
731 computeAndAddLiveIns(LiveRegs, MBB&: *LoopTailMBB);
732 computeAndAddLiveIns(LiveRegs, MBB&: *DoneMBB);
733
734 return true;
735}
736
737// If a BNE on the cmpxchg comparison result immediately follows the cmpxchg
738// operation, it can be folded into the cmpxchg expansion by
739// modifying the branch within 'LoopHead' (which performs the same
740// comparison). This is a valid transformation because after altering the
741// LoopHead's BNE destination, the BNE following the cmpxchg becomes
742// redundant and and be deleted. In the case of a masked cmpxchg, an
743// appropriate AND and BNE must be matched.
744//
745// On success, returns true and deletes the matching BNE or AND+BNE, sets the
746// LoopHeadBNETarget argument to the target that should be used within the
747// loop head, and removes that block as a successor to MBB.
748bool RISCVExpandPseudoAtomicsImpl::tryToFoldBNEOnCmpXchgResult(
749 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg,
750 Register CmpValReg, Register MaskReg,
751 MachineBasicBlock *&LoopHeadBNETarget) const {
752 SmallVector<MachineInstr *> ToErase;
753 auto E = MBB.end();
754 if (MBBI == E)
755 return false;
756 MBBI = skipDebugInstructionsForward(It: MBBI, End: E);
757
758 // If we have a masked cmpxchg, match AND dst, DestReg, MaskReg.
759 if (MaskReg.isValid()) {
760 if (MBBI == E || MBBI->getOpcode() != RISCV::AND)
761 return false;
762 Register ANDOp1 = MBBI->getOperand(i: 1).getReg();
763 Register ANDOp2 = MBBI->getOperand(i: 2).getReg();
764 if (!(ANDOp1 == DestReg && ANDOp2 == MaskReg) &&
765 !(ANDOp1 == MaskReg && ANDOp2 == DestReg))
766 return false;
767 // We now expect the BNE to use the result of the AND as an operand.
768 DestReg = MBBI->getOperand(i: 0).getReg();
769 ToErase.push_back(Elt: &*MBBI);
770 MBBI = skipDebugInstructionsForward(It: std::next(x: MBBI), End: E);
771 }
772
773 // Match BNE DestReg, MaskReg.
774 if (MBBI == E || MBBI->getOpcode() != RISCV::BNE)
775 return false;
776 Register BNEOp0 = MBBI->getOperand(i: 0).getReg();
777 Register BNEOp1 = MBBI->getOperand(i: 1).getReg();
778 if (!(BNEOp0 == DestReg && BNEOp1 == CmpValReg) &&
779 !(BNEOp0 == CmpValReg && BNEOp1 == DestReg))
780 return false;
781
782 // Make sure the branch is the only user of the AND.
783 if (MaskReg.isValid()) {
784 if (BNEOp0 == DestReg && !MBBI->getOperand(i: 0).isKill())
785 return false;
786 if (BNEOp1 == DestReg && !MBBI->getOperand(i: 1).isKill())
787 return false;
788 }
789
790 ToErase.push_back(Elt: &*MBBI);
791 LoopHeadBNETarget = MBBI->getOperand(i: 2).getMBB();
792 MBBI = skipDebugInstructionsForward(It: std::next(x: MBBI), End: E);
793 if (MBBI != E)
794 return false;
795
796 MBB.removeSuccessor(Succ: LoopHeadBNETarget);
797 for (auto *MI : ToErase)
798 MI->eraseFromParent();
799 return true;
800}
801
802bool RISCVExpandPseudoAtomicsImpl::expandAtomicCmpXchg(
803 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool IsMasked,
804 int Width, MachineBasicBlock::iterator &NextMBBI) const {
805 MachineInstr &MI = *MBBI;
806 DebugLoc DL = MI.getDebugLoc();
807 MachineFunction *MF = MBB.getParent();
808 auto LoopHeadMBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
809 auto LoopTailMBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
810 auto DoneMBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
811
812 Register DestReg = MI.getOperand(i: 0).getReg();
813 Register ScratchReg = MI.getOperand(i: 1).getReg();
814 Register AddrReg = MI.getOperand(i: 2).getReg();
815 Register CmpValReg = MI.getOperand(i: 3).getReg();
816 Register NewValReg = MI.getOperand(i: 4).getReg();
817 Register MaskReg = IsMasked ? MI.getOperand(i: 5).getReg() : Register();
818
819 MachineBasicBlock *LoopHeadBNETarget = DoneMBB;
820 tryToFoldBNEOnCmpXchgResult(MBB, MBBI: std::next(x: MBBI), DestReg, CmpValReg, MaskReg,
821 LoopHeadBNETarget);
822
823 // Insert new MBBs.
824 MF->insert(MBBI: ++MBB.getIterator(), MBB: LoopHeadMBB);
825 MF->insert(MBBI: ++LoopHeadMBB->getIterator(), MBB: LoopTailMBB);
826 MF->insert(MBBI: ++LoopTailMBB->getIterator(), MBB: DoneMBB);
827
828 // Set up successors and transfer remaining instructions to DoneMBB.
829 LoopHeadMBB->addSuccessor(Succ: LoopTailMBB);
830 LoopHeadMBB->addSuccessor(Succ: LoopHeadBNETarget);
831 LoopTailMBB->addSuccessor(Succ: DoneMBB);
832 LoopTailMBB->addSuccessor(Succ: LoopHeadMBB);
833 DoneMBB->splice(Where: DoneMBB->end(), Other: &MBB, From: MI, To: MBB.end());
834 DoneMBB->transferSuccessors(FromMBB: &MBB);
835 MBB.addSuccessor(Succ: LoopHeadMBB);
836
837 AtomicOrdering Ordering =
838 static_cast<AtomicOrdering>(MI.getOperand(i: IsMasked ? 6 : 5).getImm());
839
840 if (!IsMasked) {
841 // .loophead:
842 // lr.[w|d] dest, (addr)
843 // bne dest, cmpval, done
844 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: getLRForRMW(Ordering, Width)), DestReg)
845 .addReg(RegNo: AddrReg);
846 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BNE))
847 .addReg(RegNo: DestReg)
848 .addReg(RegNo: CmpValReg)
849 .addMBB(MBB: LoopHeadBNETarget);
850 // .looptail:
851 // sc.[w|d] scratch, newval, (addr)
852 // bnez scratch, loophead
853 BuildMI(BB: LoopTailMBB, MIMD: DL, MCID: TII->get(Opcode: getSCForRMW(Ordering, Width)), DestReg: ScratchReg)
854 .addReg(RegNo: NewValReg)
855 .addReg(RegNo: AddrReg);
856 BuildMI(BB: LoopTailMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BNE))
857 .addReg(RegNo: ScratchReg)
858 .addReg(RegNo: RISCV::X0)
859 .addMBB(MBB: LoopHeadMBB);
860 } else {
861 // .loophead:
862 // lr.w dest, (addr)
863 // and scratch, dest, mask
864 // bne scratch, cmpval, done
865 Register MaskReg = MI.getOperand(i: 5).getReg();
866 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: getLRForRMW(Ordering, Width)), DestReg)
867 .addReg(RegNo: AddrReg);
868 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::AND), DestReg: ScratchReg)
869 .addReg(RegNo: DestReg)
870 .addReg(RegNo: MaskReg);
871 BuildMI(BB: LoopHeadMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BNE))
872 .addReg(RegNo: ScratchReg)
873 .addReg(RegNo: CmpValReg)
874 .addMBB(MBB: LoopHeadBNETarget);
875
876 // .looptail:
877 // xor scratch, dest, newval
878 // and scratch, scratch, mask
879 // xor scratch, dest, scratch
880 // sc.w scratch, scratch, (adrr)
881 // bnez scratch, loophead
882 insertMaskedMerge(DL, MBB: LoopTailMBB, DestReg: ScratchReg, OldValReg: DestReg, NewValReg, MaskReg,
883 ScratchReg);
884 BuildMI(BB: LoopTailMBB, MIMD: DL, MCID: TII->get(Opcode: getSCForRMW(Ordering, Width)), DestReg: ScratchReg)
885 .addReg(RegNo: ScratchReg)
886 .addReg(RegNo: AddrReg);
887 BuildMI(BB: LoopTailMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BNE))
888 .addReg(RegNo: ScratchReg)
889 .addReg(RegNo: RISCV::X0)
890 .addMBB(MBB: LoopHeadMBB);
891 }
892
893 NextMBBI = MBB.end();
894 MI.eraseFromParent();
895
896 LivePhysRegs LiveRegs;
897 computeAndAddLiveIns(LiveRegs, MBB&: *LoopHeadMBB);
898 computeAndAddLiveIns(LiveRegs, MBB&: *LoopTailMBB);
899 computeAndAddLiveIns(LiveRegs, MBB&: *DoneMBB);
900
901 return true;
902}
903
904char RISCVExpandPseudoAtomicsLegacy::ID = 0;
905
906INITIALIZE_PASS(RISCVExpandPseudoAtomicsLegacy, "riscv-expand-pseudo-atomics",
907 RISCV_EXPAND_PSEUDO_ATOMICS_NAME, false, false)
908
909FunctionPass *llvm::createRISCVExpandPseudoAtomicsLegacyPass() {
910 return new RISCVExpandPseudoAtomicsLegacy();
911}
912
913PreservedAnalyses
914RISCVExpandPseudoAtomicsPass::run(MachineFunction &MF,
915 MachineFunctionAnalysisManager &MFAM) {
916 bool Changed = RISCVExpandPseudoAtomicsImpl().run(MF);
917 if (!Changed)
918 return PreservedAnalyses::all();
919 return getMachineFunctionPassPreservedAnalyses();
920}
921