1//===-- MipsExpandPseudoInsts.cpp - Expand pseudo instructions ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass that expands pseudo instructions into target
10// instructions to allow proper scheduling, if-conversion, and other late
11// optimizations. This pass should be run after register allocation but before
12// the post-regalloc scheduling pass.
13//
14// This is currently only used for expanding atomic pseudos after register
15// allocation. We do this to avoid the fast register allocator introducing
16// spills between ll and sc. These stores cause some MIPS implementations to
17// abort the atomic RMW sequence.
18//
19//===----------------------------------------------------------------------===//
20
21#include "Mips.h"
22#include "MipsInstrInfo.h"
23#include "MipsSubtarget.h"
24#include "llvm/CodeGen/LivePhysRegs.h"
25#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
26#include "llvm/CodeGen/MachineFunctionPass.h"
27#include "llvm/CodeGen/MachineInstrBuilder.h"
28#include "llvm/CodeGen/MachinePassManager.h"
29#include "llvm/IR/Analysis.h"
30
31using namespace llvm;
32
33#define DEBUG_TYPE "mips-expand-pseudo"
34
35namespace {
36class MipsExpandPseudoLegacy : public MachineFunctionPass {
37public:
38 static char ID;
39 MipsExpandPseudoLegacy() : MachineFunctionPass(ID) {}
40
41 bool runOnMachineFunction(MachineFunction &Fn) override;
42
43 MachineFunctionProperties getRequiredProperties() const override {
44 return MachineFunctionProperties().setNoVRegs();
45 }
46
47 StringRef getPassName() const override {
48 return "Mips pseudo instruction expansion pass";
49 }
50};
51char MipsExpandPseudoLegacy::ID = 0;
52} // namespace
53
54static bool expandAtomicCmpSwapSubword(MachineBasicBlock &BB,
55 MachineBasicBlock::iterator I,
56 MachineBasicBlock::iterator &NMBBI,
57 const MipsSubtarget *STI,
58 const MipsInstrInfo *TII) {
59
60 MachineFunction *MF = BB.getParent();
61
62 const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
63 DebugLoc DL = I->getDebugLoc();
64 unsigned LL, SC;
65
66 unsigned ZERO = Mips::R0;
67 unsigned BNE = Mips::BNE;
68 unsigned BEQ = Mips::BEQ;
69 unsigned SEOp =
70 I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I8_POSTRA ? Mips::SEB : Mips::SEH;
71
72 if (STI->inMicroMipsMode()) {
73 LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
74 SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
75 BNE = STI->hasMips32r6() ? Mips::BNEC_MMR6 : Mips::BNE_MM;
76 BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
77 } else {
78 LL = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
79 : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
80 SC = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
81 : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
82 }
83
84 Register Dest = I->getOperand(i: 0).getReg();
85 Register Ptr = I->getOperand(i: 1).getReg();
86 Register Mask = I->getOperand(i: 2).getReg();
87 Register ShiftCmpVal = I->getOperand(i: 3).getReg();
88 Register Mask2 = I->getOperand(i: 4).getReg();
89 Register ShiftNewVal = I->getOperand(i: 5).getReg();
90 Register ShiftAmnt = I->getOperand(i: 6).getReg();
91 Register Scratch = I->getOperand(i: 7).getReg();
92 Register Scratch2 = I->getOperand(i: 8).getReg();
93
94 // insert new blocks after the current block
95 const BasicBlock *LLVM_BB = BB.getBasicBlock();
96 MachineBasicBlock *loop1MBB = MF->CreateMachineBasicBlock(BB: LLVM_BB);
97 MachineBasicBlock *loop2MBB = MF->CreateMachineBasicBlock(BB: LLVM_BB);
98 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB: LLVM_BB);
99 MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(BB: LLVM_BB);
100 MachineFunction::iterator It = ++BB.getIterator();
101 MF->insert(MBBI: It, MBB: loop1MBB);
102 MF->insert(MBBI: It, MBB: loop2MBB);
103 MF->insert(MBBI: It, MBB: sinkMBB);
104 MF->insert(MBBI: It, MBB: exitMBB);
105
106 // Transfer the remainder of BB and its successor edges to exitMBB.
107 exitMBB->splice(Where: exitMBB->begin(), Other: &BB,
108 From: std::next(x: MachineBasicBlock::iterator(I)), To: BB.end());
109 exitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: &BB);
110
111 // thisMBB:
112 // ...
113 // fallthrough --> loop1MBB
114 BB.addSuccessor(Succ: loop1MBB, Prob: BranchProbability::getOne());
115 loop1MBB->addSuccessor(Succ: sinkMBB);
116 loop1MBB->addSuccessor(Succ: loop2MBB);
117 loop1MBB->normalizeSuccProbs();
118 loop2MBB->addSuccessor(Succ: loop1MBB);
119 loop2MBB->addSuccessor(Succ: sinkMBB);
120 loop2MBB->normalizeSuccProbs();
121 sinkMBB->addSuccessor(Succ: exitMBB, Prob: BranchProbability::getOne());
122
123 // loop1MBB:
124 // ll dest, 0(ptr)
125 // and Mask', dest, Mask
126 // bne Mask', ShiftCmpVal, exitMBB
127 BuildMI(BB: loop1MBB, MIMD: DL, MCID: TII->get(Opcode: LL), DestReg: Scratch).addReg(RegNo: Ptr).addImm(Val: 0);
128 BuildMI(BB: loop1MBB, MIMD: DL, MCID: TII->get(Opcode: Mips::AND), DestReg: Scratch2)
129 .addReg(RegNo: Scratch)
130 .addReg(RegNo: Mask);
131 BuildMI(BB: loop1MBB, MIMD: DL, MCID: TII->get(Opcode: BNE))
132 .addReg(RegNo: Scratch2).addReg(RegNo: ShiftCmpVal).addMBB(MBB: sinkMBB);
133
134 // loop2MBB:
135 // and dest, dest, mask2
136 // or dest, dest, ShiftNewVal
137 // sc dest, dest, 0(ptr)
138 // beq dest, $0, loop1MBB
139 BuildMI(BB: loop2MBB, MIMD: DL, MCID: TII->get(Opcode: Mips::AND), DestReg: Scratch)
140 .addReg(RegNo: Scratch, Flags: RegState::Kill)
141 .addReg(RegNo: Mask2);
142 BuildMI(BB: loop2MBB, MIMD: DL, MCID: TII->get(Opcode: Mips::OR), DestReg: Scratch)
143 .addReg(RegNo: Scratch, Flags: RegState::Kill)
144 .addReg(RegNo: ShiftNewVal);
145 BuildMI(BB: loop2MBB, MIMD: DL, MCID: TII->get(Opcode: SC), DestReg: Scratch)
146 .addReg(RegNo: Scratch, Flags: RegState::Kill)
147 .addReg(RegNo: Ptr)
148 .addImm(Val: 0);
149 BuildMI(BB: loop2MBB, MIMD: DL, MCID: TII->get(Opcode: BEQ))
150 .addReg(RegNo: Scratch, Flags: RegState::Kill)
151 .addReg(RegNo: ZERO)
152 .addMBB(MBB: loop1MBB);
153
154 // sinkMBB:
155 // srl srlres, Mask', shiftamt
156 // sign_extend dest,srlres
157 BuildMI(BB: sinkMBB, MIMD: DL, MCID: TII->get(Opcode: Mips::SRLV), DestReg: Dest)
158 .addReg(RegNo: Scratch2)
159 .addReg(RegNo: ShiftAmnt);
160 if (STI->hasMips32r2()) {
161 BuildMI(BB: sinkMBB, MIMD: DL, MCID: TII->get(Opcode: SEOp), DestReg: Dest).addReg(RegNo: Dest);
162 } else {
163 const unsigned ShiftImm =
164 I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I16_POSTRA ? 16 : 24;
165 BuildMI(BB: sinkMBB, MIMD: DL, MCID: TII->get(Opcode: Mips::SLL), DestReg: Dest)
166 .addReg(RegNo: Dest, Flags: RegState::Kill)
167 .addImm(Val: ShiftImm);
168 BuildMI(BB: sinkMBB, MIMD: DL, MCID: TII->get(Opcode: Mips::SRA), DestReg: Dest)
169 .addReg(RegNo: Dest, Flags: RegState::Kill)
170 .addImm(Val: ShiftImm);
171 }
172
173 LivePhysRegs LiveRegs;
174 computeAndAddLiveIns(LiveRegs, MBB&: *loop1MBB);
175 computeAndAddLiveIns(LiveRegs, MBB&: *loop2MBB);
176 computeAndAddLiveIns(LiveRegs, MBB&: *sinkMBB);
177 computeAndAddLiveIns(LiveRegs, MBB&: *exitMBB);
178
179 NMBBI = BB.end();
180 I->eraseFromParent();
181 return true;
182}
183
184static bool expandAtomicCmpSwap(MachineBasicBlock &BB,
185 MachineBasicBlock::iterator I,
186 MachineBasicBlock::iterator &NMBBI,
187 const MipsSubtarget *STI,
188 const MipsInstrInfo *TII) {
189
190 const unsigned Size =
191 I->getOpcode() == Mips::ATOMIC_CMP_SWAP_I32_POSTRA ? 4 : 8;
192 MachineFunction *MF = BB.getParent();
193
194 const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
195 DebugLoc DL = I->getDebugLoc();
196
197 unsigned LL, SC, ZERO, BNE, BEQ, MOVE;
198
199 if (Size == 4) {
200 if (STI->inMicroMipsMode()) {
201 LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
202 SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
203 BNE = STI->hasMips32r6() ? Mips::BNEC_MMR6 : Mips::BNE_MM;
204 BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
205 } else {
206 LL = STI->hasMips32r6()
207 ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
208 : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
209 SC = STI->hasMips32r6()
210 ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
211 : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
212 BNE = Mips::BNE;
213 BEQ = Mips::BEQ;
214 }
215
216 ZERO = Mips::R0;
217 MOVE = Mips::OR;
218 } else {
219 LL = STI->hasMips64r6() ? Mips::LLD_R6 : Mips::LLD;
220 SC = STI->hasMips64r6() ? Mips::SCD_R6 : Mips::SCD;
221 ZERO = Mips::R0_64;
222 BNE = Mips::BNE64;
223 BEQ = Mips::BEQ64;
224 MOVE = Mips::OR64;
225 }
226
227 Register Dest = I->getOperand(i: 0).getReg();
228 Register Ptr = I->getOperand(i: 1).getReg();
229 Register OldVal = I->getOperand(i: 2).getReg();
230 Register NewVal = I->getOperand(i: 3).getReg();
231 Register Scratch = I->getOperand(i: 4).getReg();
232
233 // insert new blocks after the current block
234 const BasicBlock *LLVM_BB = BB.getBasicBlock();
235 MachineBasicBlock *loop1MBB = MF->CreateMachineBasicBlock(BB: LLVM_BB);
236 MachineBasicBlock *loop2MBB = MF->CreateMachineBasicBlock(BB: LLVM_BB);
237 MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(BB: LLVM_BB);
238 MachineFunction::iterator It = ++BB.getIterator();
239 MF->insert(MBBI: It, MBB: loop1MBB);
240 MF->insert(MBBI: It, MBB: loop2MBB);
241 MF->insert(MBBI: It, MBB: exitMBB);
242
243 // Transfer the remainder of BB and its successor edges to exitMBB.
244 exitMBB->splice(Where: exitMBB->begin(), Other: &BB,
245 From: std::next(x: MachineBasicBlock::iterator(I)), To: BB.end());
246 exitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: &BB);
247
248 // thisMBB:
249 // ...
250 // fallthrough --> loop1MBB
251 BB.addSuccessor(Succ: loop1MBB, Prob: BranchProbability::getOne());
252 loop1MBB->addSuccessor(Succ: exitMBB);
253 loop1MBB->addSuccessor(Succ: loop2MBB);
254 loop1MBB->normalizeSuccProbs();
255 loop2MBB->addSuccessor(Succ: loop1MBB);
256 loop2MBB->addSuccessor(Succ: exitMBB);
257 loop2MBB->normalizeSuccProbs();
258
259 // loop1MBB:
260 // ll dest, 0(ptr)
261 // bne dest, oldval, exitMBB
262 BuildMI(BB: loop1MBB, MIMD: DL, MCID: TII->get(Opcode: LL), DestReg: Dest).addReg(RegNo: Ptr).addImm(Val: 0);
263 BuildMI(BB: loop1MBB, MIMD: DL, MCID: TII->get(Opcode: BNE))
264 .addReg(RegNo: Dest, Flags: RegState::Kill).addReg(RegNo: OldVal).addMBB(MBB: exitMBB);
265
266 // loop2MBB:
267 // move scratch, NewVal
268 // sc Scratch, Scratch, 0(ptr)
269 // beq Scratch, $0, loop1MBB
270 BuildMI(BB: loop2MBB, MIMD: DL, MCID: TII->get(Opcode: MOVE), DestReg: Scratch).addReg(RegNo: NewVal).addReg(RegNo: ZERO);
271 BuildMI(BB: loop2MBB, MIMD: DL, MCID: TII->get(Opcode: SC), DestReg: Scratch)
272 .addReg(RegNo: Scratch).addReg(RegNo: Ptr).addImm(Val: 0);
273 BuildMI(BB: loop2MBB, MIMD: DL, MCID: TII->get(Opcode: BEQ))
274 .addReg(RegNo: Scratch, Flags: RegState::Kill).addReg(RegNo: ZERO).addMBB(MBB: loop1MBB);
275
276 LivePhysRegs LiveRegs;
277 computeAndAddLiveIns(LiveRegs, MBB&: *loop1MBB);
278 computeAndAddLiveIns(LiveRegs, MBB&: *loop2MBB);
279 computeAndAddLiveIns(LiveRegs, MBB&: *exitMBB);
280
281 NMBBI = BB.end();
282 I->eraseFromParent();
283 return true;
284}
285
286static bool expandAtomicBinOpSubword(MachineBasicBlock &BB,
287 MachineBasicBlock::iterator I,
288 MachineBasicBlock::iterator &NMBBI,
289 const MipsSubtarget *STI,
290 const MipsInstrInfo *TII) {
291
292 MachineFunction *MF = BB.getParent();
293
294 const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
295 DebugLoc DL = I->getDebugLoc();
296
297 unsigned LL, SC, SLT, SLTu, OR, MOVN, MOVZ, SELNEZ, SELEQZ;
298 unsigned BEQ = Mips::BEQ;
299 unsigned SEOp = Mips::SEH;
300
301 if (STI->inMicroMipsMode()) {
302 LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
303 SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
304 BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
305 SLT = Mips::SLT_MM;
306 SLTu = Mips::SLTu_MM;
307 OR = STI->hasMips32r6() ? Mips::OR_MMR6 : Mips::OR_MM;
308 MOVN = Mips::MOVN_I_MM;
309 MOVZ = Mips::MOVZ_I_MM;
310 SELNEZ = STI->hasMips32r6() ? Mips::SELNEZ_MMR6 : Mips::SELNEZ;
311 SELEQZ = STI->hasMips32r6() ? Mips::SELEQZ_MMR6 : Mips::SELEQZ;
312 } else {
313 LL = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
314 : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
315 SC = STI->hasMips32r6() ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
316 : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
317 SLT = Mips::SLT;
318 SLTu = Mips::SLTu;
319 OR = Mips::OR;
320 MOVN = Mips::MOVN_I_I;
321 MOVZ = Mips::MOVZ_I_I;
322 SELNEZ = Mips::SELNEZ;
323 SELEQZ = Mips::SELEQZ;
324 }
325
326 bool IsSwap = false;
327 bool IsNand = false;
328 bool IsMin = false;
329 bool IsMax = false;
330 bool IsUnsigned = false;
331 bool DestOK = false;
332
333 unsigned Opcode = 0;
334 switch (I->getOpcode()) {
335 case Mips::ATOMIC_LOAD_NAND_I8_POSTRA:
336 SEOp = Mips::SEB;
337 [[fallthrough]];
338 case Mips::ATOMIC_LOAD_NAND_I16_POSTRA:
339 IsNand = true;
340 break;
341 case Mips::ATOMIC_SWAP_I8_POSTRA:
342 SEOp = Mips::SEB;
343 [[fallthrough]];
344 case Mips::ATOMIC_SWAP_I16_POSTRA:
345 IsSwap = true;
346 break;
347 case Mips::ATOMIC_LOAD_ADD_I8_POSTRA:
348 SEOp = Mips::SEB;
349 [[fallthrough]];
350 case Mips::ATOMIC_LOAD_ADD_I16_POSTRA:
351 Opcode = Mips::ADDu;
352 break;
353 case Mips::ATOMIC_LOAD_SUB_I8_POSTRA:
354 SEOp = Mips::SEB;
355 [[fallthrough]];
356 case Mips::ATOMIC_LOAD_SUB_I16_POSTRA:
357 Opcode = Mips::SUBu;
358 break;
359 case Mips::ATOMIC_LOAD_AND_I8_POSTRA:
360 SEOp = Mips::SEB;
361 [[fallthrough]];
362 case Mips::ATOMIC_LOAD_AND_I16_POSTRA:
363 Opcode = Mips::AND;
364 break;
365 case Mips::ATOMIC_LOAD_OR_I8_POSTRA:
366 SEOp = Mips::SEB;
367 [[fallthrough]];
368 case Mips::ATOMIC_LOAD_OR_I16_POSTRA:
369 Opcode = Mips::OR;
370 break;
371 case Mips::ATOMIC_LOAD_XOR_I8_POSTRA:
372 SEOp = Mips::SEB;
373 [[fallthrough]];
374 case Mips::ATOMIC_LOAD_XOR_I16_POSTRA:
375 Opcode = Mips::XOR;
376 break;
377 case Mips::ATOMIC_LOAD_UMIN_I8_POSTRA:
378 SEOp = Mips::SEB;
379 IsUnsigned = true;
380 IsMin = true;
381 break;
382 case Mips::ATOMIC_LOAD_UMIN_I16_POSTRA:
383 IsUnsigned = true;
384 IsMin = true;
385 break;
386 case Mips::ATOMIC_LOAD_MIN_I8_POSTRA:
387 SEOp = Mips::SEB;
388 IsMin = true;
389 break;
390 case Mips::ATOMIC_LOAD_MIN_I16_POSTRA:
391 IsMin = true;
392 break;
393 case Mips::ATOMIC_LOAD_UMAX_I8_POSTRA:
394 SEOp = Mips::SEB;
395 IsUnsigned = true;
396 IsMax = true;
397 break;
398 case Mips::ATOMIC_LOAD_UMAX_I16_POSTRA:
399 IsUnsigned = true;
400 IsMax = true;
401 break;
402 case Mips::ATOMIC_LOAD_MAX_I8_POSTRA:
403 SEOp = Mips::SEB;
404 IsMax = true;
405 break;
406 case Mips::ATOMIC_LOAD_MAX_I16_POSTRA:
407 IsMax = true;
408 break;
409 default:
410 llvm_unreachable("Unknown subword atomic pseudo for expansion!");
411 }
412
413 Register Dest = I->getOperand(i: 0).getReg();
414 Register Ptr = I->getOperand(i: 1).getReg();
415 Register Incr = I->getOperand(i: 2).getReg();
416 Register Mask = I->getOperand(i: 3).getReg();
417 Register Mask2 = I->getOperand(i: 4).getReg();
418 Register ShiftAmnt = I->getOperand(i: 5).getReg();
419 Register OldVal = I->getOperand(i: 6).getReg();
420 Register BinOpRes = I->getOperand(i: 7).getReg();
421 Register StoreVal = I->getOperand(i: 8).getReg();
422 bool NoMovnInstr = (IsMin || IsMax) && !STI->hasMips4() && !STI->hasMips32();
423
424 const BasicBlock *LLVM_BB = BB.getBasicBlock();
425 MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(BB: LLVM_BB);
426 MachineBasicBlock *loop1MBB = nullptr;
427 MachineBasicBlock *loop2MBB = nullptr;
428 if (NoMovnInstr) {
429 loop1MBB = MF->CreateMachineBasicBlock(BB: LLVM_BB);
430 loop2MBB = MF->CreateMachineBasicBlock(BB: LLVM_BB);
431 }
432 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB: LLVM_BB);
433 MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(BB: LLVM_BB);
434 MachineFunction::iterator It = ++BB.getIterator();
435 MF->insert(MBBI: It, MBB: loopMBB);
436 if (NoMovnInstr) {
437 MF->insert(MBBI: It, MBB: loop1MBB);
438 MF->insert(MBBI: It, MBB: loop2MBB);
439 }
440 MF->insert(MBBI: It, MBB: sinkMBB);
441 MF->insert(MBBI: It, MBB: exitMBB);
442
443 exitMBB->splice(Where: exitMBB->begin(), Other: &BB, From: std::next(x: I), To: BB.end());
444 exitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: &BB);
445
446 BB.addSuccessor(Succ: loopMBB, Prob: BranchProbability::getOne());
447 if (NoMovnInstr) {
448 loopMBB->addSuccessor(Succ: loop1MBB);
449 loopMBB->addSuccessor(Succ: loop2MBB);
450 } else {
451 loopMBB->addSuccessor(Succ: sinkMBB);
452 loopMBB->addSuccessor(Succ: loopMBB);
453 loopMBB->normalizeSuccProbs();
454 }
455 if (NoMovnInstr) {
456 loop1MBB->addSuccessor(Succ: loop2MBB);
457 loop2MBB->addSuccessor(Succ: loopMBB);
458 loop2MBB->addSuccessor(Succ: sinkMBB);
459 }
460
461 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: LL), DestReg: OldVal).addReg(RegNo: Ptr).addImm(Val: 0);
462 if (IsNand) {
463 // and andres, oldval, incr2
464 // nor binopres, $0, andres
465 // and newval, binopres, mask
466 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: Mips::AND), DestReg: BinOpRes)
467 .addReg(RegNo: OldVal)
468 .addReg(RegNo: Incr);
469 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: Mips::NOR), DestReg: BinOpRes)
470 .addReg(RegNo: Mips::R0)
471 .addReg(RegNo: BinOpRes);
472 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: Mips::AND), DestReg: BinOpRes)
473 .addReg(RegNo: BinOpRes)
474 .addReg(RegNo: Mask);
475 } else if (IsMin || IsMax) {
476
477 assert(I->getNumOperands() == 10 &&
478 "Atomics min|max|umin|umax use an additional register");
479 Register Scratch4 = I->getOperand(i: 9).getReg();
480
481 unsigned SLTScratch4 = IsUnsigned ? SLTu : SLT;
482 unsigned SELIncr = IsMax ? SELNEZ : SELEQZ;
483 unsigned SELOldVal = IsMax ? SELEQZ : SELNEZ;
484 unsigned MOVIncr = IsMax ? MOVN : MOVZ;
485
486 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: Mips::SRAV), DestReg: StoreVal)
487 .addReg(RegNo: OldVal)
488 .addReg(RegNo: ShiftAmnt);
489 if (IsUnsigned) {
490 const unsigned OpMask = SEOp == Mips::SEH ? 0xffff : 0xff;
491 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: Mips::ANDi), DestReg: StoreVal)
492 .addReg(RegNo: StoreVal)
493 .addImm(Val: OpMask);
494 } else if (STI->hasMips32r2()) {
495 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: SEOp), DestReg: StoreVal).addReg(RegNo: StoreVal);
496 } else {
497 const unsigned ShiftImm = SEOp == Mips::SEH ? 16 : 24;
498 const unsigned SROp = IsUnsigned ? Mips::SRL : Mips::SRA;
499 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: Mips::SLL), DestReg: StoreVal)
500 .addReg(RegNo: StoreVal, Flags: RegState::Kill)
501 .addImm(Val: ShiftImm);
502 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: SROp), DestReg: StoreVal)
503 .addReg(RegNo: StoreVal, Flags: RegState::Kill)
504 .addImm(Val: ShiftImm);
505 }
506 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: Mips::OR), DestReg: Dest)
507 .addReg(RegNo: Mips::R0)
508 .addReg(RegNo: StoreVal);
509 DestOK = true;
510 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: Mips::SLLV), DestReg: StoreVal)
511 .addReg(RegNo: StoreVal)
512 .addReg(RegNo: ShiftAmnt);
513
514 // unsigned: sltu Scratch4, StoreVal, Incr
515 // signed: slt Scratch4, StoreVal, Incr
516 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: SLTScratch4), DestReg: Scratch4)
517 .addReg(RegNo: StoreVal)
518 .addReg(RegNo: Incr);
519
520 if (STI->hasMips64r6() || STI->hasMips32r6()) {
521 // max: seleqz BinOpRes, OldVal, Scratch4
522 // selnez Scratch4, Incr, Scratch4
523 // or BinOpRes, BinOpRes, Scratch4
524 // min: selnqz BinOpRes, OldVal, Scratch4
525 // seleqz Scratch4, Incr, Scratch4
526 // or BinOpRes, BinOpRes, Scratch4
527 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: SELOldVal), DestReg: BinOpRes)
528 .addReg(RegNo: StoreVal)
529 .addReg(RegNo: Scratch4);
530 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: SELIncr), DestReg: Scratch4)
531 .addReg(RegNo: Incr)
532 .addReg(RegNo: Scratch4);
533 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: OR), DestReg: BinOpRes)
534 .addReg(RegNo: BinOpRes)
535 .addReg(RegNo: Scratch4);
536 } else if (STI->hasMips4() || STI->hasMips32()) {
537 // max: move BinOpRes, StoreVal
538 // movn BinOpRes, Incr, Scratch4, BinOpRes
539 // min: move BinOpRes, StoreVal
540 // movz BinOpRes, Incr, Scratch4, BinOpRes
541 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: OR), DestReg: BinOpRes)
542 .addReg(RegNo: StoreVal)
543 .addReg(RegNo: Mips::R0);
544 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: MOVIncr), DestReg: BinOpRes)
545 .addReg(RegNo: Incr)
546 .addReg(RegNo: Scratch4)
547 .addReg(RegNo: BinOpRes);
548 } else {
549 // if min:
550 // loopMBB: move BinOpRes, StoreVal
551 // bne Scratch4, 0, loop2MBB
552 // loop1MBB: move BinOpRes, Incr
553 // loop2MBB: and BinOpRes, BinOpRes, Mask
554 // and StoreVal, OlddVal, Mask2
555 // or StoreVal, StoreVal, BinOpRes
556 // StoreVal<tied1> = sc StoreVal, 0(Ptr)
557 // beq StoreVal, zero, loopMBB
558 //
559 // if max:
560 // loopMBB: move BinOpRes, Incr
561 // bne Scratch4, 0, loop2MBB
562 // loop1MBB: move BinOpRes, StoreVal
563 // loop2MBB: and BinOpRes, BinOpRes, Mask
564 // and StoreVal, OlddVal, Mask2
565 // or StoreVal, StoreVal, BinOpRes
566 // StoreVal<tied1> = sc StoreVal, 0(Ptr)
567 // beq StoreVal, zero, loopMBB
568 if (IsMin) {
569 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: OR), DestReg: BinOpRes)
570 .addReg(RegNo: StoreVal)
571 .addReg(RegNo: Mips::R0);
572 BuildMI(BB: loop1MBB, MIMD: DL, MCID: TII->get(Opcode: OR), DestReg: BinOpRes)
573 .addReg(RegNo: Incr)
574 .addReg(RegNo: Mips::R0);
575 } else {
576 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: OR), DestReg: BinOpRes)
577 .addReg(RegNo: Incr)
578 .addReg(RegNo: Mips::R0);
579 BuildMI(BB: loop1MBB, MIMD: DL, MCID: TII->get(Opcode: OR), DestReg: BinOpRes)
580 .addReg(RegNo: StoreVal)
581 .addReg(RegNo: Mips::R0);
582 }
583 // loop1MBB is laid out directly after loopMBB, so fall through to it
584 // rather than using an unconditional jump.
585 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: Mips::BNE))
586 .addReg(RegNo: Scratch4)
587 .addReg(RegNo: Mips::R0)
588 .addMBB(MBB: loop2MBB);
589 }
590
591 // and BinOpRes, BinOpRes, Mask
592 if (NoMovnInstr)
593 BuildMI(BB: loop2MBB, MIMD: DL, MCID: TII->get(Opcode: Mips::AND), DestReg: BinOpRes)
594 .addReg(RegNo: BinOpRes)
595 .addReg(RegNo: Mask);
596 else
597 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: Mips::AND), DestReg: BinOpRes)
598 .addReg(RegNo: BinOpRes)
599 .addReg(RegNo: Mask);
600
601 } else if (!IsSwap) {
602 // <binop> binopres, oldval, incr2
603 // and newval, binopres, mask
604 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode), DestReg: BinOpRes)
605 .addReg(RegNo: OldVal)
606 .addReg(RegNo: Incr);
607 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: Mips::AND), DestReg: BinOpRes)
608 .addReg(RegNo: BinOpRes)
609 .addReg(RegNo: Mask);
610 } else { // atomic.swap
611 // and newval, incr2, mask
612 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: Mips::AND), DestReg: BinOpRes)
613 .addReg(RegNo: Incr)
614 .addReg(RegNo: Mask);
615 }
616
617 // and StoreVal, OlddVal, Mask2
618 // or StoreVal, StoreVal, BinOpRes
619 // StoreVal<tied1> = sc StoreVal, 0(Ptr)
620 // beq StoreVal, zero, loopMBB
621 if (NoMovnInstr) {
622 BuildMI(BB: loop2MBB, MIMD: DL, MCID: TII->get(Opcode: Mips::AND), DestReg: StoreVal)
623 .addReg(RegNo: OldVal)
624 .addReg(RegNo: Mask2);
625 BuildMI(BB: loop2MBB, MIMD: DL, MCID: TII->get(Opcode: Mips::OR), DestReg: StoreVal)
626 .addReg(RegNo: StoreVal)
627 .addReg(RegNo: BinOpRes);
628 BuildMI(BB: loop2MBB, MIMD: DL, MCID: TII->get(Opcode: SC), DestReg: StoreVal)
629 .addReg(RegNo: StoreVal)
630 .addReg(RegNo: Ptr)
631 .addImm(Val: 0);
632 BuildMI(BB: loop2MBB, MIMD: DL, MCID: TII->get(Opcode: BEQ))
633 .addReg(RegNo: StoreVal)
634 .addReg(RegNo: Mips::R0)
635 .addMBB(MBB: loopMBB);
636 } else {
637 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: Mips::AND), DestReg: StoreVal)
638 .addReg(RegNo: OldVal)
639 .addReg(RegNo: Mask2);
640 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: Mips::OR), DestReg: StoreVal)
641 .addReg(RegNo: StoreVal)
642 .addReg(RegNo: BinOpRes);
643 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: SC), DestReg: StoreVal)
644 .addReg(RegNo: StoreVal)
645 .addReg(RegNo: Ptr)
646 .addImm(Val: 0);
647 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: BEQ))
648 .addReg(RegNo: StoreVal)
649 .addReg(RegNo: Mips::R0)
650 .addMBB(MBB: loopMBB);
651 }
652
653 // sinkMBB:
654 // and maskedoldval1,oldval,mask
655 // srl srlres,maskedoldval1,shiftamt
656 // sign_extend dest,srlres
657
658 if (!DestOK) {
659 sinkMBB->addSuccessor(Succ: exitMBB, Prob: BranchProbability::getOne());
660 BuildMI(BB: sinkMBB, MIMD: DL, MCID: TII->get(Opcode: Mips::AND), DestReg: Dest).addReg(RegNo: OldVal).addReg(RegNo: Mask);
661 BuildMI(BB: sinkMBB, MIMD: DL, MCID: TII->get(Opcode: Mips::SRLV), DestReg: Dest)
662 .addReg(RegNo: Dest)
663 .addReg(RegNo: ShiftAmnt);
664
665 if (STI->hasMips32r2()) {
666 BuildMI(BB: sinkMBB, MIMD: DL, MCID: TII->get(Opcode: SEOp), DestReg: Dest).addReg(RegNo: Dest);
667 } else {
668 const unsigned ShiftImm = SEOp == Mips::SEH ? 16 : 24;
669 BuildMI(BB: sinkMBB, MIMD: DL, MCID: TII->get(Opcode: Mips::SLL), DestReg: Dest)
670 .addReg(RegNo: Dest, Flags: RegState::Kill)
671 .addImm(Val: ShiftImm);
672 BuildMI(BB: sinkMBB, MIMD: DL, MCID: TII->get(Opcode: Mips::SRA), DestReg: Dest)
673 .addReg(RegNo: Dest, Flags: RegState::Kill)
674 .addImm(Val: ShiftImm);
675 }
676 }
677
678 LivePhysRegs LiveRegs;
679 computeAndAddLiveIns(LiveRegs, MBB&: *loopMBB);
680 if (loop1MBB) {
681 assert(loop2MBB && "should have 2 loop blocks");
682 computeAndAddLiveIns(LiveRegs, MBB&: *loop1MBB);
683 computeAndAddLiveIns(LiveRegs, MBB&: *loop2MBB);
684 }
685 computeAndAddLiveIns(LiveRegs, MBB&: *sinkMBB);
686 computeAndAddLiveIns(LiveRegs, MBB&: *exitMBB);
687
688 NMBBI = BB.end();
689 I->eraseFromParent();
690
691 return true;
692}
693
694static bool expandAtomicBinOp(MachineBasicBlock &BB,
695 MachineBasicBlock::iterator I,
696 MachineBasicBlock::iterator &NMBBI, unsigned Size,
697 const MipsSubtarget *STI,
698 const MipsInstrInfo *TII) {
699 MachineFunction *MF = BB.getParent();
700
701 const bool ArePtrs64bit = STI->getABI().ArePtrs64bit();
702 DebugLoc DL = I->getDebugLoc();
703
704 unsigned LL, SC, ZERO, BEQ, SLT, SLTu, OR, MOVN, MOVZ, SELNEZ, SELEQZ;
705
706 if (Size == 4) {
707 if (STI->inMicroMipsMode()) {
708 LL = STI->hasMips32r6() ? Mips::LL_MMR6 : Mips::LL_MM;
709 SC = STI->hasMips32r6() ? Mips::SC_MMR6 : Mips::SC_MM;
710 BEQ = STI->hasMips32r6() ? Mips::BEQC_MMR6 : Mips::BEQ_MM;
711 SLT = Mips::SLT_MM;
712 SLTu = Mips::SLTu_MM;
713 OR = STI->hasMips32r6() ? Mips::OR_MMR6 : Mips::OR_MM;
714 MOVN = Mips::MOVN_I_MM;
715 MOVZ = Mips::MOVZ_I_MM;
716 SELNEZ = STI->hasMips32r6() ? Mips::SELNEZ_MMR6 : Mips::SELNEZ;
717 SELEQZ = STI->hasMips32r6() ? Mips::SELEQZ_MMR6 : Mips::SELEQZ;
718 } else {
719 LL = STI->hasMips32r6()
720 ? (ArePtrs64bit ? Mips::LL64_R6 : Mips::LL_R6)
721 : (ArePtrs64bit ? Mips::LL64 : Mips::LL);
722 SC = STI->hasMips32r6()
723 ? (ArePtrs64bit ? Mips::SC64_R6 : Mips::SC_R6)
724 : (ArePtrs64bit ? Mips::SC64 : Mips::SC);
725 BEQ = Mips::BEQ;
726 SLT = Mips::SLT;
727 SLTu = Mips::SLTu;
728 OR = Mips::OR;
729 MOVN = Mips::MOVN_I_I;
730 MOVZ = Mips::MOVZ_I_I;
731 SELNEZ = Mips::SELNEZ;
732 SELEQZ = Mips::SELEQZ;
733 }
734
735 ZERO = Mips::R0;
736 } else {
737 LL = STI->hasMips64r6() ? Mips::LLD_R6 : Mips::LLD;
738 SC = STI->hasMips64r6() ? Mips::SCD_R6 : Mips::SCD;
739 ZERO = Mips::R0_64;
740 BEQ = Mips::BEQ64;
741 SLT = Mips::SLT64;
742 SLTu = Mips::SLTu64;
743 OR = Mips::OR64;
744 MOVN = Mips::MOVN_I64_I64;
745 MOVZ = Mips::MOVZ_I64_I64;
746 SELNEZ = Mips::SELNEZ64;
747 SELEQZ = Mips::SELEQZ64;
748 }
749
750 Register OldVal = I->getOperand(i: 0).getReg();
751 Register Ptr = I->getOperand(i: 1).getReg();
752 Register Incr = I->getOperand(i: 2).getReg();
753 Register Scratch = I->getOperand(i: 3).getReg();
754
755 unsigned Opcode = 0;
756 unsigned AND = 0;
757 unsigned NOR = 0;
758
759 bool IsOr = false;
760 bool IsNand = false;
761 bool IsMin = false;
762 bool IsMax = false;
763 bool IsUnsigned = false;
764
765 switch (I->getOpcode()) {
766 case Mips::ATOMIC_LOAD_ADD_I32_POSTRA:
767 Opcode = Mips::ADDu;
768 break;
769 case Mips::ATOMIC_LOAD_SUB_I32_POSTRA:
770 Opcode = Mips::SUBu;
771 break;
772 case Mips::ATOMIC_LOAD_AND_I32_POSTRA:
773 Opcode = Mips::AND;
774 break;
775 case Mips::ATOMIC_LOAD_OR_I32_POSTRA:
776 Opcode = Mips::OR;
777 break;
778 case Mips::ATOMIC_LOAD_XOR_I32_POSTRA:
779 Opcode = Mips::XOR;
780 break;
781 case Mips::ATOMIC_LOAD_NAND_I32_POSTRA:
782 IsNand = true;
783 AND = Mips::AND;
784 NOR = Mips::NOR;
785 break;
786 case Mips::ATOMIC_SWAP_I32_POSTRA:
787 IsOr = true;
788 break;
789 case Mips::ATOMIC_LOAD_ADD_I64_POSTRA:
790 Opcode = Mips::DADDu;
791 break;
792 case Mips::ATOMIC_LOAD_SUB_I64_POSTRA:
793 Opcode = Mips::DSUBu;
794 break;
795 case Mips::ATOMIC_LOAD_AND_I64_POSTRA:
796 Opcode = Mips::AND64;
797 break;
798 case Mips::ATOMIC_LOAD_OR_I64_POSTRA:
799 Opcode = Mips::OR64;
800 break;
801 case Mips::ATOMIC_LOAD_XOR_I64_POSTRA:
802 Opcode = Mips::XOR64;
803 break;
804 case Mips::ATOMIC_LOAD_NAND_I64_POSTRA:
805 IsNand = true;
806 AND = Mips::AND64;
807 NOR = Mips::NOR64;
808 break;
809 case Mips::ATOMIC_SWAP_I64_POSTRA:
810 IsOr = true;
811 break;
812 case Mips::ATOMIC_LOAD_UMIN_I32_POSTRA:
813 case Mips::ATOMIC_LOAD_UMIN_I64_POSTRA:
814 IsUnsigned = true;
815 [[fallthrough]];
816 case Mips::ATOMIC_LOAD_MIN_I32_POSTRA:
817 case Mips::ATOMIC_LOAD_MIN_I64_POSTRA:
818 IsMin = true;
819 break;
820 case Mips::ATOMIC_LOAD_UMAX_I32_POSTRA:
821 case Mips::ATOMIC_LOAD_UMAX_I64_POSTRA:
822 IsUnsigned = true;
823 [[fallthrough]];
824 case Mips::ATOMIC_LOAD_MAX_I32_POSTRA:
825 case Mips::ATOMIC_LOAD_MAX_I64_POSTRA:
826 IsMax = true;
827 break;
828 default:
829 llvm_unreachable("Unknown pseudo atomic!");
830 }
831
832 bool NoMovnInstr = (IsMin || IsMax) && !STI->hasMips4() && !STI->hasMips32();
833 const BasicBlock *LLVM_BB = BB.getBasicBlock();
834 MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(BB: LLVM_BB);
835 MachineBasicBlock *loop1MBB = nullptr;
836 MachineBasicBlock *loop2MBB = nullptr;
837 if (NoMovnInstr) {
838 loop1MBB = MF->CreateMachineBasicBlock(BB: LLVM_BB);
839 loop2MBB = MF->CreateMachineBasicBlock(BB: LLVM_BB);
840 }
841 MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(BB: LLVM_BB);
842 MachineFunction::iterator It = ++BB.getIterator();
843 MF->insert(MBBI: It, MBB: loopMBB);
844 if (NoMovnInstr) {
845 MF->insert(MBBI: It, MBB: loop1MBB);
846 MF->insert(MBBI: It, MBB: loop2MBB);
847 }
848 MF->insert(MBBI: It, MBB: exitMBB);
849
850 exitMBB->splice(Where: exitMBB->begin(), Other: &BB, From: std::next(x: I), To: BB.end());
851 exitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: &BB);
852
853 BB.addSuccessor(Succ: loopMBB, Prob: BranchProbability::getOne());
854 if (NoMovnInstr) {
855 loopMBB->addSuccessor(Succ: loop1MBB);
856 loopMBB->addSuccessor(Succ: loop2MBB);
857 } else {
858 loopMBB->addSuccessor(Succ: exitMBB);
859 loopMBB->addSuccessor(Succ: loopMBB);
860 }
861 loopMBB->normalizeSuccProbs();
862 if (NoMovnInstr) {
863 loop1MBB->addSuccessor(Succ: loop2MBB);
864 loop2MBB->addSuccessor(Succ: loopMBB);
865 loop2MBB->addSuccessor(Succ: exitMBB);
866 }
867
868 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: LL), DestReg: OldVal).addReg(RegNo: Ptr).addImm(Val: 0);
869 assert((OldVal != Ptr) && "Clobbered the wrong ptr reg!");
870 assert((OldVal != Incr) && "Clobbered the wrong reg!");
871 if (IsMin || IsMax) {
872 // Remove trailing kill/dead register operands added by
873 // MachineInstr::addRegisterKilled. These are super-register markers that
874 // must correspond to one of the physical registers in operands 1-4.
875 // The kill/dead markers may also appear on preceding subregs.
876 const TargetRegisterInfo *TRI = STI->getRegisterInfo();
877 while (I->getNumOperands() > 5) {
878 auto &Op = I->getOperand(i: I->getNumOperands() - 1);
879 // Check if this register overlaps with any physical register in
880 // operands 1-4 that has kill/dead marker (i.e., it's a super-register
881 // marker for subregs).
882 bool HasOverlapWithKill = false;
883 for (unsigned i = 1; i <= 4; ++i) {
884 auto &RefOp = I->getOperand(i);
885 if (RefOp.isReg() && RefOp.getReg().isPhysical() &&
886 TRI->regsOverlap(RegA: Op.getReg(), RegB: RefOp.getReg()) &&
887 (RefOp.isKill() || RefOp.isDead())) {
888 HasOverlapWithKill = true;
889 break;
890 }
891 }
892 // Remove if HasOverlapWithKill is true or Op has kill/dead marker.
893 bool HasKillOrDead = Op.isReg() && Op.getReg().isPhysical() &&
894 (Op.isKill() || Op.isDead());
895 if (!HasOverlapWithKill && !HasKillOrDead)
896 break;
897 I->removeOperand(OpNo: I->getNumOperands() - 1);
898 }
899
900 assert(I->getNumOperands() == 5 &&
901 "Atomics min|max|umin|umax use an additional register");
902 MCRegister Scratch2 = I->getOperand(i: 4).getReg().asMCReg();
903
904 // On Mips64 result of slt is GPR32.
905 MCRegister Scratch2_32 =
906 (Size == 8) ? STI->getRegisterInfo()->getSubReg(Reg: Scratch2, Idx: Mips::sub_32)
907 : Scratch2;
908
909 unsigned SLTScratch2 = IsUnsigned ? SLTu : SLT;
910 unsigned SELIncr = IsMax ? SELNEZ : SELEQZ;
911 unsigned SELOldVal = IsMax ? SELEQZ : SELNEZ;
912 unsigned MOVIncr = IsMax ? MOVN : MOVZ;
913
914 // unsigned: sltu Scratch2, oldVal, Incr
915 // signed: slt Scratch2, oldVal, Incr
916 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: SLTScratch2), DestReg: Scratch2_32)
917 .addReg(RegNo: OldVal)
918 .addReg(RegNo: Incr);
919
920 if (STI->hasMips64r6() || STI->hasMips32r6()) {
921 // max: seleqz Scratch, OldVal, Scratch2
922 // selnez Scratch2, Incr, Scratch2
923 // or Scratch, Scratch, Scratch2
924 // min: selnez Scratch, OldVal, Scratch2
925 // seleqz Scratch2, Incr, Scratch2
926 // or Scratch, Scratch, Scratch2
927 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: SELOldVal), DestReg: Scratch)
928 .addReg(RegNo: OldVal)
929 .addReg(RegNo: Scratch2);
930 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: SELIncr), DestReg: Scratch2)
931 .addReg(RegNo: Incr)
932 .addReg(RegNo: Scratch2);
933 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: OR), DestReg: Scratch)
934 .addReg(RegNo: Scratch)
935 .addReg(RegNo: Scratch2);
936 } else if (STI->hasMips4() || STI->hasMips32()) {
937 // max: move Scratch, OldVal
938 // movn Scratch, Incr, Scratch2, Scratch
939 // min: move Scratch, OldVal
940 // movz Scratch, Incr, Scratch2, Scratch
941 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: OR), DestReg: Scratch)
942 .addReg(RegNo: OldVal)
943 .addReg(RegNo: ZERO);
944 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: MOVIncr), DestReg: Scratch)
945 .addReg(RegNo: Incr)
946 .addReg(RegNo: Scratch2)
947 .addReg(RegNo: Scratch);
948 } else {
949 // if min:
950 // loopMBB: move Scratch, OldVal
951 // bne Scratch2_32, 0, loop2MBB
952 // loop1MBB: move Scratch, Incr
953 // loop2MBB: sc $2, 0($4)
954 // beqz $2, $BB0_1
955 // nop
956 //
957 // if max:
958 // loopMBB: move Scratch, Incr
959 // bne Scratch2_32, 0, loop2MBB
960 // loop1MBB: move Scratch, OldVal
961 // loop2MBB: sc $2, 0($4)
962 // beqz $2, $BB0_1
963 // nop
964 if (IsMin) {
965 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: OR), DestReg: Scratch).addReg(RegNo: OldVal).addReg(RegNo: ZERO);
966 BuildMI(BB: loop1MBB, MIMD: DL, MCID: TII->get(Opcode: OR), DestReg: Scratch).addReg(RegNo: Incr).addReg(RegNo: ZERO);
967 } else {
968 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: OR), DestReg: Scratch).addReg(RegNo: Incr).addReg(RegNo: ZERO);
969 BuildMI(BB: loop1MBB, MIMD: DL, MCID: TII->get(Opcode: OR), DestReg: Scratch)
970 .addReg(RegNo: OldVal)
971 .addReg(RegNo: ZERO);
972 }
973 // Scratch2_32 is always a GPR32, so use the 32-bit BNE even for the
974 // 64-bit atomics. loop1MBB is laid out directly after loopMBB, so fall
975 // through to it rather than using an unconditional jump.
976 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: Mips::BNE))
977 .addReg(RegNo: Scratch2_32)
978 .addReg(RegNo: Mips::R0)
979 .addMBB(MBB: loop2MBB);
980 }
981
982 } else if (Opcode) {
983 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode), DestReg: Scratch).addReg(RegNo: OldVal).addReg(RegNo: Incr);
984 } else if (IsNand) {
985 assert(AND && NOR &&
986 "Unknown nand instruction for atomic pseudo expansion");
987 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: AND), DestReg: Scratch).addReg(RegNo: OldVal).addReg(RegNo: Incr);
988 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: NOR), DestReg: Scratch).addReg(RegNo: ZERO).addReg(RegNo: Scratch);
989 } else {
990 assert(IsOr && OR && "Unknown instruction for atomic pseudo expansion!");
991 (void)IsOr;
992 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: OR), DestReg: Scratch).addReg(RegNo: Incr).addReg(RegNo: ZERO);
993 }
994
995 if (NoMovnInstr) {
996 BuildMI(BB: loop2MBB, MIMD: DL, MCID: TII->get(Opcode: SC), DestReg: Scratch)
997 .addReg(RegNo: Scratch)
998 .addReg(RegNo: Ptr)
999 .addImm(Val: 0);
1000 BuildMI(BB: loop2MBB, MIMD: DL, MCID: TII->get(Opcode: BEQ))
1001 .addReg(RegNo: Scratch)
1002 .addReg(RegNo: ZERO)
1003 .addMBB(MBB: loopMBB);
1004 } else {
1005 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: SC), DestReg: Scratch)
1006 .addReg(RegNo: Scratch)
1007 .addReg(RegNo: Ptr)
1008 .addImm(Val: 0);
1009 BuildMI(BB: loopMBB, MIMD: DL, MCID: TII->get(Opcode: BEQ))
1010 .addReg(RegNo: Scratch)
1011 .addReg(RegNo: ZERO)
1012 .addMBB(MBB: loopMBB);
1013 }
1014
1015 NMBBI = BB.end();
1016 I->eraseFromParent();
1017
1018 LivePhysRegs LiveRegs;
1019 computeAndAddLiveIns(LiveRegs, MBB&: *loopMBB);
1020 if (loop1MBB) {
1021 assert(loop2MBB && "should have 2 loop blocks");
1022 computeAndAddLiveIns(LiveRegs, MBB&: *loop1MBB);
1023 computeAndAddLiveIns(LiveRegs, MBB&: *loop2MBB);
1024 }
1025 computeAndAddLiveIns(LiveRegs, MBB&: *exitMBB);
1026
1027 return true;
1028}
1029
1030static bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
1031 MachineBasicBlock::iterator &NMBB,
1032 const MipsSubtarget *STI, const MipsInstrInfo *TII) {
1033
1034 bool Modified = false;
1035
1036 switch (MBBI->getOpcode()) {
1037 case Mips::ATOMIC_CMP_SWAP_I32_POSTRA:
1038 case Mips::ATOMIC_CMP_SWAP_I64_POSTRA:
1039 return expandAtomicCmpSwap(BB&: MBB, I: MBBI, NMBBI&: NMBB, STI, TII);
1040 case Mips::ATOMIC_CMP_SWAP_I8_POSTRA:
1041 case Mips::ATOMIC_CMP_SWAP_I16_POSTRA:
1042 return expandAtomicCmpSwapSubword(BB&: MBB, I: MBBI, NMBBI&: NMBB, STI, TII);
1043 case Mips::ATOMIC_SWAP_I8_POSTRA:
1044 case Mips::ATOMIC_SWAP_I16_POSTRA:
1045 case Mips::ATOMIC_LOAD_NAND_I8_POSTRA:
1046 case Mips::ATOMIC_LOAD_NAND_I16_POSTRA:
1047 case Mips::ATOMIC_LOAD_ADD_I8_POSTRA:
1048 case Mips::ATOMIC_LOAD_ADD_I16_POSTRA:
1049 case Mips::ATOMIC_LOAD_SUB_I8_POSTRA:
1050 case Mips::ATOMIC_LOAD_SUB_I16_POSTRA:
1051 case Mips::ATOMIC_LOAD_AND_I8_POSTRA:
1052 case Mips::ATOMIC_LOAD_AND_I16_POSTRA:
1053 case Mips::ATOMIC_LOAD_OR_I8_POSTRA:
1054 case Mips::ATOMIC_LOAD_OR_I16_POSTRA:
1055 case Mips::ATOMIC_LOAD_XOR_I8_POSTRA:
1056 case Mips::ATOMIC_LOAD_XOR_I16_POSTRA:
1057 case Mips::ATOMIC_LOAD_MIN_I8_POSTRA:
1058 case Mips::ATOMIC_LOAD_MIN_I16_POSTRA:
1059 case Mips::ATOMIC_LOAD_MAX_I8_POSTRA:
1060 case Mips::ATOMIC_LOAD_MAX_I16_POSTRA:
1061 case Mips::ATOMIC_LOAD_UMIN_I8_POSTRA:
1062 case Mips::ATOMIC_LOAD_UMIN_I16_POSTRA:
1063 case Mips::ATOMIC_LOAD_UMAX_I8_POSTRA:
1064 case Mips::ATOMIC_LOAD_UMAX_I16_POSTRA:
1065 return expandAtomicBinOpSubword(BB&: MBB, I: MBBI, NMBBI&: NMBB, STI, TII);
1066 case Mips::ATOMIC_LOAD_ADD_I32_POSTRA:
1067 case Mips::ATOMIC_LOAD_SUB_I32_POSTRA:
1068 case Mips::ATOMIC_LOAD_AND_I32_POSTRA:
1069 case Mips::ATOMIC_LOAD_OR_I32_POSTRA:
1070 case Mips::ATOMIC_LOAD_XOR_I32_POSTRA:
1071 case Mips::ATOMIC_LOAD_NAND_I32_POSTRA:
1072 case Mips::ATOMIC_SWAP_I32_POSTRA:
1073 case Mips::ATOMIC_LOAD_MIN_I32_POSTRA:
1074 case Mips::ATOMIC_LOAD_MAX_I32_POSTRA:
1075 case Mips::ATOMIC_LOAD_UMIN_I32_POSTRA:
1076 case Mips::ATOMIC_LOAD_UMAX_I32_POSTRA:
1077 return expandAtomicBinOp(BB&: MBB, I: MBBI, NMBBI&: NMBB, Size: 4, STI, TII);
1078 case Mips::ATOMIC_LOAD_ADD_I64_POSTRA:
1079 case Mips::ATOMIC_LOAD_SUB_I64_POSTRA:
1080 case Mips::ATOMIC_LOAD_AND_I64_POSTRA:
1081 case Mips::ATOMIC_LOAD_OR_I64_POSTRA:
1082 case Mips::ATOMIC_LOAD_XOR_I64_POSTRA:
1083 case Mips::ATOMIC_LOAD_NAND_I64_POSTRA:
1084 case Mips::ATOMIC_SWAP_I64_POSTRA:
1085 case Mips::ATOMIC_LOAD_MIN_I64_POSTRA:
1086 case Mips::ATOMIC_LOAD_MAX_I64_POSTRA:
1087 case Mips::ATOMIC_LOAD_UMIN_I64_POSTRA:
1088 case Mips::ATOMIC_LOAD_UMAX_I64_POSTRA:
1089 return expandAtomicBinOp(BB&: MBB, I: MBBI, NMBBI&: NMBB, Size: 8, STI, TII);
1090 default:
1091 return Modified;
1092 }
1093}
1094
1095static bool expandMBB(MachineBasicBlock &MBB, const MipsSubtarget *STI,
1096 const MipsInstrInfo *TII) {
1097 bool Modified = false;
1098
1099 MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
1100 while (MBBI != E) {
1101 MachineBasicBlock::iterator NMBBI = std::next(x: MBBI);
1102 Modified |= expandMI(MBB, MBBI, NMBB&: NMBBI, STI, TII);
1103 MBBI = NMBBI;
1104 }
1105
1106 return Modified;
1107}
1108
1109static bool expandPseudosInMF(MachineFunction &MF) {
1110 const MipsSubtarget *STI = &MF.getSubtarget<MipsSubtarget>();
1111 const MipsInstrInfo *TII = STI->getInstrInfo();
1112
1113 bool Modified = false;
1114 for (MachineBasicBlock &MBB : MF)
1115 Modified |= expandMBB(MBB, STI, TII);
1116
1117 if (Modified)
1118 MF.RenumberBlocks();
1119
1120 return Modified;
1121}
1122
1123/// createMipsExpandPseudoPass - returns an instance of the pseudo instruction
1124/// expansion pass.
1125FunctionPass *llvm::createMipsExpandPseudoLegacyPass() {
1126 return new MipsExpandPseudoLegacy();
1127}
1128
1129bool MipsExpandPseudoLegacy::runOnMachineFunction(MachineFunction &MF) {
1130 return expandPseudosInMF(MF);
1131}
1132
1133PreservedAnalyses
1134MipsExpandPseudoPass::run(MachineFunction &MF,
1135 MachineFunctionAnalysisManager &MFAM) {
1136 return expandPseudosInMF(MF) ? getMachineFunctionPassPreservedAnalyses()
1137 : PreservedAnalyses::all();
1138}
1139