1//===- RISCVOptWInstrs.cpp - MI W instruction optimizations ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===---------------------------------------------------------------------===//
8//
9// This pass does some optimizations for *W instructions at the MI level.
10//
11// First it removes unneeded sext.w instructions. Either because the sign
12// extended bits aren't consumed or because the input was already sign extended
13// by an earlier instruction.
14//
15// Then:
16// 1. Unless explicit disabled or the target prefers instructions with W suffix,
17// it removes the -w suffix from opw instructions whenever all users are
18// dependent only on the lower word of the result of the instruction.
19// The cases handled are:
20// * addw because c.add has a larger register encoding than c.addw.
21// * addiw because it helps reduce test differences between RV32 and RV64
22// w/o being a pessimization.
23// * mulw because c.mulw doesn't exist but c.mul does (w/ zcb)
24// * slliw because c.slliw doesn't exist and c.slli does
25//
26// 2. Or if explicit enabled or the target prefers instructions with W suffix,
27// it adds the W suffix to the instruction whenever all users are dependent
28// only on the lower word of the result of the instruction.
29// The cases handled are:
30// * add/addi/sub/mul.
31// * slli with imm < 32.
32// * ld/lwu.
33//===---------------------------------------------------------------------===//
34
35#include "RISCV.h"
36#include "RISCVMachineFunctionInfo.h"
37#include "RISCVSubtarget.h"
38#include "llvm/ADT/SmallSet.h"
39#include "llvm/ADT/Statistic.h"
40#include "llvm/CodeGen/MachineFunctionPass.h"
41#include "llvm/CodeGen/TargetInstrInfo.h"
42
43using namespace llvm;
44
45#define DEBUG_TYPE "riscv-opt-w-instrs"
46#define RISCV_OPT_W_INSTRS_NAME "RISC-V Optimize W Instructions"
47
48STATISTIC(NumRemovedSExtW, "Number of removed sign-extensions");
49STATISTIC(NumTransformedToWInstrs,
50 "Number of instructions transformed to W-ops");
51STATISTIC(NumTransformedToNonWInstrs,
52 "Number of instructions transformed to non-W-ops");
53
54static cl::opt<bool> DisableSExtWRemoval("riscv-disable-sextw-removal",
55 cl::desc("Disable removal of sext.w"),
56 cl::init(Val: false), cl::Hidden);
57static cl::opt<bool> DisableStripWSuffix("riscv-disable-strip-w-suffix",
58 cl::desc("Disable strip W suffix"),
59 cl::init(Val: false), cl::Hidden);
60
61namespace {
62
63class RISCVOptWInstrsImpl {
64public:
65 bool run(MachineFunction &MF);
66
67private:
68 bool removeSExtWInstrs(MachineFunction &MF, const RISCVInstrInfo &TII,
69 const RISCVSubtarget &ST, MachineRegisterInfo &MRI);
70 bool canonicalizeWSuffixes(MachineFunction &MF, const RISCVInstrInfo &TII,
71 const RISCVSubtarget &ST,
72 MachineRegisterInfo &MRI);
73};
74
75class RISCVOptWInstrsLegacy : public MachineFunctionPass {
76public:
77 static char ID;
78
79 RISCVOptWInstrsLegacy() : MachineFunctionPass(ID) {}
80
81 bool runOnMachineFunction(MachineFunction &MF) override;
82
83 void getAnalysisUsage(AnalysisUsage &AU) const override {
84 AU.setPreservesCFG();
85 MachineFunctionPass::getAnalysisUsage(AU);
86 }
87
88 StringRef getPassName() const override { return RISCV_OPT_W_INSTRS_NAME; }
89};
90
91} // end anonymous namespace
92
93char RISCVOptWInstrsLegacy::ID = 0;
94INITIALIZE_PASS(RISCVOptWInstrsLegacy, DEBUG_TYPE, RISCV_OPT_W_INSTRS_NAME,
95 false, false)
96
97FunctionPass *llvm::createRISCVOptWInstrsLegacyPass() {
98 return new RISCVOptWInstrsLegacy();
99}
100
101static bool vectorPseudoHasAllNBitUsers(const MachineInstr &MI, unsigned OpIdx,
102 unsigned Bits) {
103 unsigned MCOpcode = RISCV::getRVVMCOpcode(RVVPseudoOpcode: MI.getOpcode());
104
105 if (!MCOpcode)
106 return false;
107
108 const MCInstrDesc &MCID = MI.getDesc();
109 const uint64_t TSFlags = MCID.TSFlags;
110 if (!RISCVII::hasSEWOp(TSFlags))
111 return false;
112 assert(RISCVII::hasVLOp(TSFlags));
113 const unsigned Log2SEW = MI.getOperand(i: RISCVII::getSEWOpNum(Desc: MCID)).getImm();
114
115 if (OpIdx == RISCVII::getVLOpNum(Desc: MCID))
116 return false;
117
118 auto NumDemandedBits =
119 RISCV::getVectorLowDemandedScalarBits(Opcode: MCOpcode, Log2SEW);
120 return NumDemandedBits && Bits >= *NumDemandedBits;
121}
122
123// Checks if all users only demand the lower \p OrigBits of the original
124// instruction's result.
125// TODO: handle multiple interdependent transformations
126static bool hasAllNBitUsers(const MachineInstr &OrigMI,
127 const RISCVSubtarget &ST,
128 const MachineRegisterInfo &MRI, unsigned OrigBits) {
129
130 SmallSet<std::pair<const MachineInstr *, unsigned>, 4> Visited;
131 SmallVector<std::pair<const MachineInstr *, unsigned>, 4> Worklist;
132
133 Worklist.emplace_back(Args: &OrigMI, Args&: OrigBits);
134
135 while (!Worklist.empty()) {
136 auto P = Worklist.pop_back_val();
137 const MachineInstr *MI = P.first;
138 unsigned Bits = P.second;
139
140 if (!Visited.insert(V: P).second)
141 continue;
142
143 // Only handle instructions with one def.
144 if (MI->getNumExplicitDefs() != 1)
145 return false;
146
147 Register DestReg = MI->getOperand(i: 0).getReg();
148 if (!DestReg.isVirtual())
149 return false;
150
151 for (auto &UserOp : MRI.use_nodbg_operands(Reg: DestReg)) {
152 const MachineInstr *UserMI = UserOp.getParent();
153 unsigned OpIdx = UserOp.getOperandNo();
154
155 switch (UserMI->getOpcode()) {
156 default:
157 if (vectorPseudoHasAllNBitUsers(MI: *UserMI, OpIdx, Bits))
158 break;
159 return false;
160
161 case RISCV::ADDIW:
162 case RISCV::ADDW:
163 case RISCV::DIVUW:
164 case RISCV::DIVW:
165 case RISCV::MULW:
166 case RISCV::REMUW:
167 case RISCV::REMW:
168 case RISCV::SLLW:
169 case RISCV::SRAIW:
170 case RISCV::SRAW:
171 case RISCV::SRLIW:
172 case RISCV::SRLW:
173 case RISCV::SUBW:
174 case RISCV::ROLW:
175 case RISCV::RORW:
176 case RISCV::RORIW:
177 case RISCV::CLSW:
178 case RISCV::CLZW:
179 case RISCV::CTZW:
180 case RISCV::CPOPW:
181 case RISCV::SLLI_UW:
182 case RISCV::ABSW:
183 case RISCV::FMV_W_X:
184 case RISCV::FCVT_H_W:
185 case RISCV::FCVT_H_W_INX:
186 case RISCV::FCVT_H_WU:
187 case RISCV::FCVT_H_WU_INX:
188 case RISCV::FCVT_S_W:
189 case RISCV::FCVT_S_W_INX:
190 case RISCV::FCVT_S_WU:
191 case RISCV::FCVT_S_WU_INX:
192 case RISCV::FCVT_D_W:
193 case RISCV::FCVT_D_W_INX:
194 case RISCV::FCVT_D_WU:
195 case RISCV::FCVT_D_WU_INX:
196 if (Bits >= 32)
197 break;
198 return false;
199
200 case RISCV::SEXT_B:
201 case RISCV::PACKH:
202 if (Bits >= 8)
203 break;
204 return false;
205 case RISCV::SEXT_H:
206 case RISCV::FMV_H_X:
207 case RISCV::ZEXT_H_RV32:
208 case RISCV::ZEXT_H_RV64:
209 case RISCV::PACKW:
210 if (Bits >= 16)
211 break;
212 return false;
213
214 case RISCV::PACK:
215 if (Bits >= (ST.getXLen() / 2))
216 break;
217 return false;
218
219 case RISCV::SRLI: {
220 // If we are shifting right by less than Bits, and users don't demand
221 // any bits that were shifted into [Bits-1:0], then we can consider this
222 // as an N-Bit user.
223 unsigned ShAmt = UserMI->getOperand(i: 2).getImm();
224 if (Bits > ShAmt) {
225 Worklist.emplace_back(Args&: UserMI, Args: Bits - ShAmt);
226 break;
227 }
228 return false;
229 }
230
231 // these overwrite higher input bits, otherwise the lower word of output
232 // depends only on the lower word of input. So check their uses read W.
233 case RISCV::SLLI: {
234 unsigned ShAmt = UserMI->getOperand(i: 2).getImm();
235 if (Bits >= (ST.getXLen() - ShAmt))
236 break;
237 Worklist.emplace_back(Args&: UserMI, Args: Bits + ShAmt);
238 break;
239 }
240 case RISCV::SLLIW: {
241 unsigned ShAmt = UserMI->getOperand(i: 2).getImm();
242 if (Bits >= 32 - ShAmt)
243 break;
244 Worklist.emplace_back(Args&: UserMI, Args: Bits + ShAmt);
245 break;
246 }
247
248 case RISCV::ANDI: {
249 uint64_t Imm = UserMI->getOperand(i: 2).getImm();
250 if (Bits >= (unsigned)llvm::bit_width(Value: Imm))
251 break;
252 Worklist.emplace_back(Args&: UserMI, Args&: Bits);
253 break;
254 }
255 case RISCV::ORI: {
256 uint64_t Imm = UserMI->getOperand(i: 2).getImm();
257 if (Bits >= (unsigned)llvm::bit_width<uint64_t>(Value: ~Imm))
258 break;
259 Worklist.emplace_back(Args&: UserMI, Args&: Bits);
260 break;
261 }
262
263 case RISCV::SLL:
264 case RISCV::BSET:
265 case RISCV::BCLR:
266 case RISCV::BINV:
267 // Operand 2 is the shift amount which uses log2(xlen) bits.
268 if (OpIdx == 2) {
269 if (Bits >= Log2_32(Value: ST.getXLen()))
270 break;
271 return false;
272 }
273 Worklist.emplace_back(Args&: UserMI, Args&: Bits);
274 break;
275
276 case RISCV::SRA:
277 case RISCV::SRL:
278 case RISCV::ROL:
279 case RISCV::ROR:
280 // Operand 2 is the shift amount which uses 6 bits.
281 if (OpIdx == 2 && Bits >= Log2_32(Value: ST.getXLen()))
282 break;
283 return false;
284
285 case RISCV::ADD_UW:
286 case RISCV::SH1ADD_UW:
287 case RISCV::SH2ADD_UW:
288 case RISCV::SH3ADD_UW:
289 // Operand 1 is implicitly zero extended.
290 if (OpIdx == 1 && Bits >= 32)
291 break;
292 Worklist.emplace_back(Args&: UserMI, Args&: Bits);
293 break;
294
295 case RISCV::BEXTI:
296 if (UserMI->getOperand(i: 2).getImm() >= Bits)
297 return false;
298 break;
299
300 case RISCV::SB:
301 // The first argument is the value to store.
302 if (OpIdx == 0 && Bits >= 8)
303 break;
304 return false;
305 case RISCV::SH:
306 // The first argument is the value to store.
307 if (OpIdx == 0 && Bits >= 16)
308 break;
309 return false;
310 case RISCV::SW:
311 // The first argument is the value to store.
312 if (OpIdx == 0 && Bits >= 32)
313 break;
314 return false;
315
316 // For these, lower word of output in these operations, depends only on
317 // the lower word of input. So, we check all uses only read lower word.
318 case RISCV::COPY:
319 case RISCV::PHI:
320
321 case RISCV::ADD:
322 case RISCV::ADDI:
323 case RISCV::AND:
324 case RISCV::MUL:
325 case RISCV::OR:
326 case RISCV::SUB:
327 case RISCV::XOR:
328 case RISCV::XORI:
329
330 case RISCV::ANDN:
331 case RISCV::CLMUL:
332 case RISCV::ORN:
333 case RISCV::SH1ADD:
334 case RISCV::SH2ADD:
335 case RISCV::SH3ADD:
336 case RISCV::XNOR:
337 case RISCV::BSETI:
338 case RISCV::BCLRI:
339 case RISCV::BINVI:
340 Worklist.emplace_back(Args&: UserMI, Args&: Bits);
341 break;
342
343 case RISCV::BREV8:
344 case RISCV::ORC_B:
345 // BREV8 and ORC_B work on bytes. Round Bits down to the nearest byte.
346 Worklist.emplace_back(Args&: UserMI, Args: alignDown(Value: Bits, Align: 8));
347 break;
348
349 case RISCV::PseudoCCMOVGPR:
350 case RISCV::PseudoCCMOVGPRNoX0:
351 // Either operand 1 or operand 2 is returned by this instruction. If
352 // only the lower word of the result is used, then only the lower word
353 // of operand 1 and 2 is used.
354 if (OpIdx != 1 && OpIdx != 2)
355 return false;
356 Worklist.emplace_back(Args&: UserMI, Args&: Bits);
357 break;
358
359 case RISCV::CZERO_EQZ:
360 case RISCV::CZERO_NEZ:
361 if (OpIdx != 1)
362 return false;
363 Worklist.emplace_back(Args&: UserMI, Args&: Bits);
364 break;
365 case RISCV::TH_EXT:
366 case RISCV::TH_EXTU:
367 unsigned Msb = UserMI->getOperand(i: 2).getImm();
368 unsigned Lsb = UserMI->getOperand(i: 3).getImm();
369 // Behavior of Msb < Lsb is not well documented.
370 if (Msb >= Lsb && Bits > Msb)
371 break;
372 return false;
373 }
374 }
375 }
376
377 return true;
378}
379
380static bool hasAllWUsers(const MachineInstr &OrigMI, const RISCVSubtarget &ST,
381 const MachineRegisterInfo &MRI) {
382 return hasAllNBitUsers(OrigMI, ST, MRI, OrigBits: 32);
383}
384
385// This function returns true if the machine instruction always outputs a value
386// where bits 63:32 match bit 31.
387static bool isSignExtendingOpW(const MachineInstr &MI, unsigned OpNo) {
388 uint64_t TSFlags = MI.getDesc().TSFlags;
389
390 // Instructions that can be determined from opcode are marked in tablegen.
391 if (TSFlags & RISCVII::IsSignExtendingOpWMask)
392 return true;
393
394 // Special cases that require checking operands.
395 switch (MI.getOpcode()) {
396 // shifting right sufficiently makes the value 32-bit sign-extended
397 case RISCV::SRAI:
398 return MI.getOperand(i: 2).getImm() >= 32;
399 case RISCV::SRLI:
400 return MI.getOperand(i: 2).getImm() > 32;
401 // The LI pattern ADDI rd, X0, imm is sign extended.
402 case RISCV::ADDI:
403 return MI.getOperand(i: 1).isReg() && MI.getOperand(i: 1).getReg() == RISCV::X0;
404 // An ANDI with an 11 bit immediate will zero bits 63:11.
405 case RISCV::ANDI:
406 return isUInt<11>(x: MI.getOperand(i: 2).getImm());
407 // An ORI with an >11 bit immediate (negative 12-bit) will set bits 63:11.
408 case RISCV::ORI:
409 return !isUInt<11>(x: MI.getOperand(i: 2).getImm());
410 // A bseti with X0 is sign extended if the immediate is less than 31.
411 case RISCV::BSETI:
412 return MI.getOperand(i: 2).getImm() < 31 &&
413 MI.getOperand(i: 1).getReg() == RISCV::X0;
414 // Copying from X0 produces zero.
415 case RISCV::COPY:
416 return MI.getOperand(i: 1).getReg() == RISCV::X0;
417 // Ignore the scratch register destination.
418 case RISCV::PseudoAtomicLoadNand32:
419 return OpNo == 0;
420 case RISCV::PseudoVMV_X_S: {
421 // vmv.x.s has at least 33 sign bits if log2(sew) <= 5.
422 int64_t Log2SEW = MI.getOperand(i: 2).getImm();
423 assert(Log2SEW >= 3 && Log2SEW <= 6 && "Unexpected Log2SEW");
424 return Log2SEW <= 5;
425 }
426 case RISCV::TH_EXT: {
427 unsigned Msb = MI.getOperand(i: 2).getImm();
428 unsigned Lsb = MI.getOperand(i: 3).getImm();
429 return Msb >= Lsb && (Msb - Lsb + 1) <= 32;
430 }
431 case RISCV::TH_EXTU: {
432 unsigned Msb = MI.getOperand(i: 2).getImm();
433 unsigned Lsb = MI.getOperand(i: 3).getImm();
434 return Msb >= Lsb && (Msb - Lsb + 1) < 32;
435 }
436 case RISCV::SATI_RV64:
437 // Saturates to signed range [-2^(imm-1), 2^(imm-1)-1].
438 // If imm <= 32, result fits in 32-bit signed range, thus sign-extended.
439 return MI.getOperand(i: 2).getImm() <= 32;
440 case RISCV::USATI_RV64:
441 // Saturates to unsigned range [0, 2^imm-1].
442 // If imm < 32, result has bit 31 clear, thus sign-extended.
443 return MI.getOperand(i: 2).getImm() < 32;
444 }
445
446 return false;
447}
448
449static bool isSignExtendedW(Register SrcReg, const RISCVSubtarget &ST,
450 const MachineRegisterInfo &MRI,
451 SmallPtrSetImpl<MachineInstr *> &FixableDef) {
452 SmallSet<Register, 4> Visited;
453 SmallVector<Register, 4> Worklist;
454
455 auto AddRegToWorkList = [&](Register SrcReg) {
456 if (!SrcReg.isVirtual())
457 return false;
458 Worklist.push_back(Elt: SrcReg);
459 return true;
460 };
461
462 if (!AddRegToWorkList(SrcReg))
463 return false;
464
465 while (!Worklist.empty()) {
466 Register Reg = Worklist.pop_back_val();
467
468 // If we already visited this register, we don't need to check it again.
469 if (!Visited.insert(V: Reg).second)
470 continue;
471
472 MachineInstr *MI = MRI.getVRegDef(Reg);
473 if (!MI)
474 continue;
475
476 int OpNo = MI->findRegisterDefOperandIdx(Reg, /*TRI=*/nullptr);
477 assert(OpNo != -1 && "Couldn't find register");
478
479 // If this is a sign extending operation we don't need to look any further.
480 if (isSignExtendingOpW(MI: *MI, OpNo))
481 continue;
482
483 // Is this an instruction that propagates sign extend?
484 switch (MI->getOpcode()) {
485 default:
486 // Unknown opcode, give up.
487 return false;
488 case RISCV::COPY: {
489 const MachineFunction *MF = MI->getMF();
490 const RISCVMachineFunctionInfo *RVFI =
491 MF->getInfo<RISCVMachineFunctionInfo>();
492
493 // If this is the entry block, see if we know the copied argument register
494 // is sign extended.
495 if (MI->getParent() == &MF->front() &&
496 RVFI->isSExt32Register(Reg: MI->getOperand(i: 0).getReg()))
497 continue;
498
499 Register CopySrcReg = MI->getOperand(i: 1).getReg();
500 if (CopySrcReg == RISCV::X10) {
501 // For a method return value, we check the ZExt/SExt flags in attribute.
502 // We assume the following code sequence for method call.
503 // PseudoCALL @bar, ...
504 // ADJCALLSTACKUP 0, 0, implicit-def dead $x2, implicit $x2
505 // %0:gpr = COPY $x10
506 //
507 // We use the PseudoCall to look up the IR function being called to find
508 // its return attributes.
509 const MachineBasicBlock *MBB = MI->getParent();
510 auto II = MI->getIterator();
511 if (II == MBB->instr_begin() ||
512 (--II)->getOpcode() != RISCV::ADJCALLSTACKUP)
513 return false;
514
515 const MachineInstr &CallMI = *(--II);
516 if (!CallMI.isCall() || !CallMI.getOperand(i: 0).isGlobal())
517 return false;
518
519 auto *CalleeFn =
520 dyn_cast_if_present<Function>(Val: CallMI.getOperand(i: 0).getGlobal());
521 if (!CalleeFn)
522 return false;
523
524 auto *IntTy = dyn_cast<IntegerType>(Val: CalleeFn->getReturnType());
525 if (!IntTy)
526 return false;
527
528 const AttributeSet &Attrs = CalleeFn->getAttributes().getRetAttrs();
529 unsigned BitWidth = IntTy->getBitWidth();
530 if ((BitWidth <= 32 && Attrs.hasAttribute(Kind: Attribute::SExt)) ||
531 (BitWidth < 32 && Attrs.hasAttribute(Kind: Attribute::ZExt)))
532 continue;
533 }
534
535 if (!AddRegToWorkList(CopySrcReg))
536 return false;
537
538 break;
539 }
540
541 // For these, we just need to check if the 1st operand is sign extended.
542 case RISCV::BCLRI:
543 case RISCV::BINVI:
544 case RISCV::BSETI:
545 if (MI->getOperand(i: 2).getImm() >= 31)
546 return false;
547 [[fallthrough]];
548 case RISCV::REM:
549 case RISCV::ANDI:
550 case RISCV::ORI:
551 case RISCV::XORI:
552 case RISCV::SRAI:
553 // |Remainder| is always <= |Dividend|. If D is 32-bit, then so is R.
554 // DIV doesn't work because of the edge case 0xf..f 8000 0000 / (long)-1
555 // Logical operations use a sign extended 12-bit immediate.
556 // Arithmetic shift right can only increase the number of sign bits.
557 if (!AddRegToWorkList(MI->getOperand(i: 1).getReg()))
558 return false;
559
560 break;
561 case RISCV::PseudoCCADDW:
562 case RISCV::PseudoCCADDIW:
563 case RISCV::PseudoCCSUBW:
564 case RISCV::PseudoCCSLLW:
565 case RISCV::PseudoCCSRLW:
566 case RISCV::PseudoCCSRAW:
567 case RISCV::PseudoCCSLLIW:
568 case RISCV::PseudoCCSRLIW:
569 case RISCV::PseudoCCSRAIW:
570 // Returns operand 1 or an ADDW/SUBW/etc. of operands 2 and 3. We only
571 // need to check if operand 1 is sign extended.
572 if (!AddRegToWorkList(MI->getOperand(i: 1).getReg()))
573 return false;
574 break;
575 case RISCV::REMU:
576 case RISCV::AND:
577 case RISCV::OR:
578 case RISCV::XOR:
579 case RISCV::ANDN:
580 case RISCV::ORN:
581 case RISCV::XNOR:
582 case RISCV::MAX:
583 case RISCV::MAXU:
584 case RISCV::MIN:
585 case RISCV::MINU:
586 case RISCV::PseudoCCMOVGPR:
587 case RISCV::PseudoCCMOVGPRNoX0:
588 case RISCV::PseudoCCAND:
589 case RISCV::PseudoCCOR:
590 case RISCV::PseudoCCXOR:
591 case RISCV::PseudoCCANDN:
592 case RISCV::PseudoCCORN:
593 case RISCV::PseudoCCXNOR:
594 case RISCV::PHI:
595 case RISCV::MERGE:
596 case RISCV::MVM:
597 case RISCV::MVMN: {
598 // If all incoming values are sign-extended, the output of AND, OR, XOR,
599 // MIN, MAX, PHI, or bitwise merge instructions is also sign-extended.
600
601 // The input registers for PHI are operand 1, 3, ...
602 // The input registers for PseudoCCMOVGPR(NoX0) are 1 and 2.
603 // The input registers for PseudoCCAND/OR/XOR are 1, 2, and 3.
604 // The input registers for MERGE/MVM/MVMN are 1, 2, and 3.
605 // The input registers for others are operand 1 and 2.
606 unsigned B = 1, E = 3, D = 1;
607 switch (MI->getOpcode()) {
608 case RISCV::PHI:
609 E = MI->getNumOperands();
610 D = 2;
611 break;
612 case RISCV::PseudoCCMOVGPR:
613 case RISCV::PseudoCCMOVGPRNoX0:
614 B = 1;
615 E = 3;
616 break;
617 case RISCV::PseudoCCAND:
618 case RISCV::PseudoCCOR:
619 case RISCV::PseudoCCXOR:
620 case RISCV::PseudoCCANDN:
621 case RISCV::PseudoCCORN:
622 case RISCV::PseudoCCXNOR:
623 B = 1;
624 E = 4;
625 break;
626 case RISCV::MERGE:
627 case RISCV::MVM:
628 case RISCV::MVMN:
629 B = 1;
630 E = 4;
631 break;
632 }
633
634 for (unsigned I = B; I != E; I += D) {
635 if (!MI->getOperand(i: I).isReg())
636 return false;
637
638 if (!AddRegToWorkList(MI->getOperand(i: I).getReg()))
639 return false;
640 }
641
642 break;
643 }
644
645 case RISCV::CZERO_EQZ:
646 case RISCV::CZERO_NEZ:
647 // Instructions return zero or operand 1. Result is sign extended if
648 // operand 1 is sign extended.
649 if (!AddRegToWorkList(MI->getOperand(i: 1).getReg()))
650 return false;
651 break;
652
653 case RISCV::ADDI: {
654 if (MI->getOperand(i: 1).isReg() && MI->getOperand(i: 1).getReg().isVirtual()) {
655 if (MachineInstr *SrcMI = MRI.getVRegDef(Reg: MI->getOperand(i: 1).getReg())) {
656 if (SrcMI->getOpcode() == RISCV::LUI &&
657 SrcMI->getOperand(i: 1).isImm()) {
658 uint64_t Imm = SrcMI->getOperand(i: 1).getImm();
659 Imm = SignExtend64<32>(x: Imm << 12);
660 Imm += (uint64_t)MI->getOperand(i: 2).getImm();
661 if (isInt<32>(x: Imm))
662 continue;
663 }
664 }
665 }
666
667 if (hasAllWUsers(OrigMI: *MI, ST, MRI)) {
668 FixableDef.insert(Ptr: MI);
669 break;
670 }
671 return false;
672 }
673
674 case RISCV::LD:
675 case RISCV::LXD: {
676 if (MI->hasOneMemOperand() && !(*MI->memoperands_begin())->isVolatile() &&
677 hasAllWUsers(OrigMI: *MI, ST, MRI)) {
678 FixableDef.insert(Ptr: MI);
679 break;
680 }
681 return false;
682 }
683
684 // With these opcode, we can "fix" them with the W-version
685 // if we know all users of the result only rely on bits 31:0
686 case RISCV::SLLI:
687 // SLLIW reads the lowest 5 bits, while SLLI reads lowest 6 bits
688 if (MI->getOperand(i: 2).getImm() >= 32)
689 return false;
690 [[fallthrough]];
691 case RISCV::ADD:
692 case RISCV::LWU:
693 case RISCV::LXWU:
694 case RISCV::MUL:
695 case RISCV::SUB:
696 if (hasAllWUsers(OrigMI: *MI, ST, MRI)) {
697 FixableDef.insert(Ptr: MI);
698 break;
699 }
700 return false;
701 case RISCV::ADD_UW:
702 // ZEXT.W is fixable to SEXT.W.
703 // TODO: In some cases it is better to delete the ZEXT.W and fix something
704 // earlier in the graph.
705 if (!MI->getOperand(i: 2).isReg() || MI->getOperand(i: 2).getReg() != RISCV::X0)
706 return false;
707
708 if (hasAllWUsers(OrigMI: *MI, ST, MRI)) {
709 FixableDef.insert(Ptr: MI);
710 break;
711 }
712 return false;
713 }
714 }
715
716 // If we get here, then every node we visited produces a sign extended value
717 // or propagated sign extended values. So the result must be sign extended.
718 return true;
719}
720
721static unsigned getWOp(unsigned Opcode) {
722 switch (Opcode) {
723 case RISCV::ADDI:
724 return RISCV::ADDIW;
725 case RISCV::ADD:
726 return RISCV::ADDW;
727 case RISCV::LD:
728 case RISCV::LWU:
729 return RISCV::LW;
730 case RISCV::LXD:
731 case RISCV::LXWU:
732 return RISCV::LXW;
733 case RISCV::MUL:
734 return RISCV::MULW;
735 case RISCV::SLLI:
736 return RISCV::SLLIW;
737 case RISCV::SUB:
738 return RISCV::SUBW;
739 default:
740 llvm_unreachable("Unexpected opcode for replacement with W variant");
741 }
742}
743
744bool RISCVOptWInstrsImpl::removeSExtWInstrs(MachineFunction &MF,
745 const RISCVInstrInfo &TII,
746 const RISCVSubtarget &ST,
747 MachineRegisterInfo &MRI) {
748 if (DisableSExtWRemoval)
749 return false;
750
751 bool MadeChange = false;
752 for (MachineBasicBlock &MBB : MF) {
753 for (MachineInstr &MI : llvm::make_early_inc_range(Range&: MBB)) {
754 // We're looking for the sext.w pattern ADDIW rd, rs1, 0.
755 if (!RISCVInstrInfo::isSEXT_W(MI))
756 continue;
757
758 Register SrcReg = MI.getOperand(i: 1).getReg();
759
760 SmallPtrSet<MachineInstr *, 4> FixableDefs;
761
762 // If all users only use the lower bits, this sext.w is redundant.
763 // Or if all definitions reaching MI sign-extend their output,
764 // then sext.w is redundant.
765 if (!hasAllWUsers(OrigMI: MI, ST, MRI) &&
766 !isSignExtendedW(SrcReg, ST, MRI, FixableDef&: FixableDefs))
767 continue;
768
769 Register DstReg = MI.getOperand(i: 0).getReg();
770 if (!MRI.constrainRegClass(Reg: SrcReg, RC: MRI.getRegClass(Reg: DstReg)))
771 continue;
772
773 // Convert Fixable instructions to their W versions.
774 for (MachineInstr *Fixable : FixableDefs) {
775 LLVM_DEBUG(dbgs() << "Replacing " << *Fixable);
776 // Convert zext.w to sext.w.
777 if (Fixable->getOpcode() == RISCV::ADD_UW) {
778 assert(Fixable->getOperand(2).isReg() &&
779 Fixable->getOperand(2).getReg() == RISCV::X0 &&
780 "Unexpected ADD_UW operand.");
781 Fixable->setDesc(TII.get(Opcode: RISCV::ADDIW));
782 Fixable->getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
783 } else {
784 Fixable->setDesc(TII.get(Opcode: getWOp(Opcode: Fixable->getOpcode())));
785 }
786 Fixable->clearFlag(Flag: MachineInstr::MIFlag::NoSWrap);
787 Fixable->clearFlag(Flag: MachineInstr::MIFlag::NoUWrap);
788 Fixable->clearFlag(Flag: MachineInstr::MIFlag::IsExact);
789 LLVM_DEBUG(dbgs() << " with " << *Fixable);
790 ++NumTransformedToWInstrs;
791 }
792
793 LLVM_DEBUG(dbgs() << "Removing redundant sign-extension\n");
794 MRI.replaceRegWith(FromReg: DstReg, ToReg: SrcReg);
795 MRI.clearKillFlags(Reg: SrcReg);
796 MI.eraseFromParent();
797 ++NumRemovedSExtW;
798 MadeChange = true;
799 }
800 }
801
802 return MadeChange;
803}
804
805// Strips or adds W suffixes to eligible instructions depending on the
806// subtarget preferences.
807bool RISCVOptWInstrsImpl::canonicalizeWSuffixes(MachineFunction &MF,
808 const RISCVInstrInfo &TII,
809 const RISCVSubtarget &ST,
810 MachineRegisterInfo &MRI) {
811 bool ShouldStripW = !(DisableStripWSuffix || ST.preferWInst());
812 bool ShouldPreferW = ST.preferWInst();
813 bool MadeChange = false;
814
815 for (MachineBasicBlock &MBB : MF) {
816 for (MachineInstr &MI : MBB) {
817 std::optional<unsigned> WOpc;
818 std::optional<unsigned> NonWOpc;
819 unsigned OrigOpc = MI.getOpcode();
820 switch (OrigOpc) {
821 default:
822 continue;
823 case RISCV::ADDW:
824 NonWOpc = RISCV::ADD;
825 break;
826 case RISCV::ADDIW:
827 NonWOpc = RISCV::ADDI;
828 break;
829 case RISCV::MULW:
830 NonWOpc = RISCV::MUL;
831 break;
832 case RISCV::SLLIW:
833 NonWOpc = RISCV::SLLI;
834 break;
835 case RISCV::SUBW:
836 NonWOpc = RISCV::SUB;
837 break;
838 case RISCV::ADD:
839 WOpc = RISCV::ADDW;
840 break;
841 case RISCV::ADDI:
842 WOpc = RISCV::ADDIW;
843 break;
844 case RISCV::SUB:
845 WOpc = RISCV::SUBW;
846 break;
847 case RISCV::MUL:
848 WOpc = RISCV::MULW;
849 break;
850 case RISCV::SLLI:
851 // SLLIW reads the lowest 5 bits, while SLLI reads lowest 6 bits.
852 if (MI.getOperand(i: 2).getImm() >= 32)
853 continue;
854 WOpc = RISCV::SLLIW;
855 break;
856 case RISCV::LD:
857 if (!MI.hasOneMemOperand() || (*MI.memoperands_begin())->isVolatile())
858 continue;
859 WOpc = RISCV::LW;
860 break;
861 case RISCV::LWU:
862 WOpc = RISCV::LW;
863 break;
864 case RISCV::LXD:
865 if (!MI.hasOneMemOperand() || (*MI.memoperands_begin())->isVolatile())
866 continue;
867 WOpc = RISCV::LXW;
868 break;
869 case RISCV::LXWU:
870 WOpc = RISCV::LXW;
871 break;
872 }
873
874 if (ShouldStripW && NonWOpc.has_value() && hasAllWUsers(OrigMI: MI, ST, MRI)) {
875 LLVM_DEBUG(dbgs() << "Replacing " << MI);
876 MI.setDesc(TII.get(Opcode: NonWOpc.value()));
877 LLVM_DEBUG(dbgs() << " with " << MI);
878 ++NumTransformedToNonWInstrs;
879 MadeChange = true;
880 continue;
881 }
882 // LWU is always converted to LW when possible as 1) LW is compressible
883 // and 2) it helps minimise differences vs RV32.
884 if ((ShouldPreferW || OrigOpc == RISCV::LWU) && WOpc.has_value() &&
885 hasAllWUsers(OrigMI: MI, ST, MRI)) {
886 LLVM_DEBUG(dbgs() << "Replacing " << MI);
887 MI.setDesc(TII.get(Opcode: WOpc.value()));
888 MI.clearFlag(Flag: MachineInstr::MIFlag::NoSWrap);
889 MI.clearFlag(Flag: MachineInstr::MIFlag::NoUWrap);
890 MI.clearFlag(Flag: MachineInstr::MIFlag::IsExact);
891 LLVM_DEBUG(dbgs() << " with " << MI);
892 ++NumTransformedToWInstrs;
893 MadeChange = true;
894 continue;
895 }
896 }
897 }
898 return MadeChange;
899}
900
901bool RISCVOptWInstrsImpl::run(MachineFunction &MF) {
902 MachineRegisterInfo &MRI = MF.getRegInfo();
903 const RISCVSubtarget &ST = MF.getSubtarget<RISCVSubtarget>();
904 const RISCVInstrInfo &TII = *ST.getInstrInfo();
905
906 if (!ST.is64Bit())
907 return false;
908
909 bool MadeChange = false;
910 MadeChange |= removeSExtWInstrs(MF, TII, ST, MRI);
911 MadeChange |= canonicalizeWSuffixes(MF, TII, ST, MRI);
912 return MadeChange;
913}
914
915bool RISCVOptWInstrsLegacy::runOnMachineFunction(MachineFunction &MF) {
916 if (skipFunction(F: MF.getFunction()))
917 return false;
918 return RISCVOptWInstrsImpl().run(MF);
919}
920
921PreservedAnalyses
922RISCVOptWInstrsPass::run(MachineFunction &MF,
923 MachineFunctionAnalysisManager &MFAM) {
924 bool Changed = RISCVOptWInstrsImpl().run(MF);
925 if (!Changed)
926 return PreservedAnalyses::all();
927
928 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
929 PA.preserveSet<CFGAnalyses>();
930 return PA;
931}
932