1//===-- RISCVInstrInfo.cpp - RISC-V Instruction Information -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the RISC-V implementation of the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RISCVInstrInfo.h"
14#include "MCTargetDesc/RISCVBaseInfo.h"
15#include "MCTargetDesc/RISCVMatInt.h"
16#include "RISCV.h"
17#include "RISCVMachineFunctionInfo.h"
18#include "RISCVSubtarget.h"
19#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/ADT/Statistic.h"
22#include "llvm/Analysis/MemoryLocation.h"
23#include "llvm/Analysis/ValueTracking.h"
24#include "llvm/CodeGen/LiveIntervals.h"
25#include "llvm/CodeGen/MachineCombinerPattern.h"
26#include "llvm/CodeGen/MachineInstrBuilder.h"
27#include "llvm/CodeGen/MachineRegisterInfo.h"
28#include "llvm/CodeGen/MachineTraceMetrics.h"
29#include "llvm/CodeGen/RegisterScavenging.h"
30#include "llvm/CodeGen/StackMaps.h"
31#include "llvm/IR/DebugInfoMetadata.h"
32#include "llvm/IR/Module.h"
33#include "llvm/MC/MCDwarf.h"
34#include "llvm/MC/MCInstBuilder.h"
35#include "llvm/MC/TargetRegistry.h"
36#include "llvm/Support/ErrorHandling.h"
37
38using namespace llvm;
39
40#define GEN_CHECK_COMPRESS_INSTR
41#include "RISCVGenCompressInstEmitter.inc"
42
43#define GET_INSTRINFO_CTOR_DTOR
44#include "RISCVGenInstrInfo.inc"
45
46#define DEBUG_TYPE "riscv-instr-info"
47STATISTIC(NumVRegSpilled,
48 "Number of registers within vector register groups spilled");
49STATISTIC(NumVRegReloaded,
50 "Number of registers within vector register groups reloaded");
51
52namespace llvm::RISCVVPseudosTable {
53
54using namespace RISCV;
55
56#define GET_RISCVVPseudosTable_IMPL
57#include "RISCVGenSearchableTables.inc"
58
59} // namespace llvm::RISCVVPseudosTable
60
61namespace llvm::RISCV {
62
63#define GET_RISCVMaskedPseudosTable_IMPL
64#include "RISCVGenSearchableTables.inc"
65
66} // end namespace llvm::RISCV
67
68RISCVInstrInfo::RISCVInstrInfo(const RISCVSubtarget &STI)
69 : RISCVGenInstrInfo(STI, RegInfo, RISCV::ADJCALLSTACKDOWN,
70 RISCV::ADJCALLSTACKUP),
71 RegInfo(STI.getHwMode()), STI(STI) {}
72
73#define GET_INSTRINFO_HELPERS
74#include "RISCVGenInstrInfo.inc"
75
76MCInst RISCVInstrInfo::getNop() const {
77 if (STI.hasStdExtZca())
78 return MCInstBuilder(RISCV::C_NOP);
79 return MCInstBuilder(RISCV::ADDI)
80 .addReg(Reg: RISCV::X0)
81 .addReg(Reg: RISCV::X0)
82 .addImm(Val: 0);
83}
84
85Register RISCVInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
86 int &FrameIndex) const {
87 TypeSize Dummy = TypeSize::getZero();
88 return isLoadFromStackSlot(MI, FrameIndex, MemBytes&: Dummy);
89}
90
91static std::optional<unsigned> getLMULForRVVWholeLoadStore(unsigned Opcode) {
92 switch (Opcode) {
93 default:
94 return std::nullopt;
95 case RISCV::VS1R_V:
96 case RISCV::VL1RE8_V:
97 case RISCV::VL1RE16_V:
98 case RISCV::VL1RE32_V:
99 case RISCV::VL1RE64_V:
100 return 1;
101 case RISCV::VS2R_V:
102 case RISCV::VL2RE8_V:
103 case RISCV::VL2RE16_V:
104 case RISCV::VL2RE32_V:
105 case RISCV::VL2RE64_V:
106 return 2;
107 case RISCV::VS4R_V:
108 case RISCV::VL4RE8_V:
109 case RISCV::VL4RE16_V:
110 case RISCV::VL4RE32_V:
111 case RISCV::VL4RE64_V:
112 return 4;
113 case RISCV::VS8R_V:
114 case RISCV::VL8RE8_V:
115 case RISCV::VL8RE16_V:
116 case RISCV::VL8RE32_V:
117 case RISCV::VL8RE64_V:
118 return 8;
119 }
120}
121
122Register RISCVInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
123 int &FrameIndex,
124 TypeSize &MemBytes) const {
125 switch (MI.getOpcode()) {
126 default:
127 return 0;
128 case RISCV::LB:
129 case RISCV::LBU:
130 MemBytes = TypeSize::getFixed(ExactSize: 1);
131 break;
132 case RISCV::LH:
133 case RISCV::LH_INX:
134 case RISCV::LHU:
135 case RISCV::FLH:
136 MemBytes = TypeSize::getFixed(ExactSize: 2);
137 break;
138 case RISCV::LW:
139 case RISCV::LW_INX:
140 case RISCV::FLW:
141 case RISCV::LWU:
142 MemBytes = TypeSize::getFixed(ExactSize: 4);
143 break;
144 case RISCV::LD:
145 case RISCV::LD_RV32:
146 case RISCV::FLD:
147 MemBytes = TypeSize::getFixed(ExactSize: 8);
148 break;
149 case RISCV::VL1RE8_V:
150 case RISCV::VL2RE8_V:
151 case RISCV::VL4RE8_V:
152 case RISCV::VL8RE8_V:
153 if (!MI.getOperand(i: 1).isFI())
154 return Register();
155 FrameIndex = MI.getOperand(i: 1).getIndex();
156 unsigned LMUL = *getLMULForRVVWholeLoadStore(Opcode: MI.getOpcode());
157 MemBytes = TypeSize::getScalable(MinimumSize: RISCV::RVVBytesPerBlock * LMUL);
158 return MI.getOperand(i: 0).getReg();
159 }
160
161 if (MI.getOperand(i: 1).isFI() && MI.getOperand(i: 2).isImm() &&
162 MI.getOperand(i: 2).getImm() == 0) {
163 FrameIndex = MI.getOperand(i: 1).getIndex();
164 return MI.getOperand(i: 0).getReg();
165 }
166
167 return 0;
168}
169
170Register RISCVInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
171 int &FrameIndex) const {
172 TypeSize Dummy = TypeSize::getZero();
173 return isStoreToStackSlot(MI, FrameIndex, MemBytes&: Dummy);
174}
175
176Register RISCVInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
177 int &FrameIndex,
178 TypeSize &MemBytes) const {
179 switch (MI.getOpcode()) {
180 default:
181 return 0;
182 case RISCV::SB:
183 MemBytes = TypeSize::getFixed(ExactSize: 1);
184 break;
185 case RISCV::SH:
186 case RISCV::SH_INX:
187 case RISCV::FSH:
188 MemBytes = TypeSize::getFixed(ExactSize: 2);
189 break;
190 case RISCV::SW:
191 case RISCV::SW_INX:
192 case RISCV::FSW:
193 MemBytes = TypeSize::getFixed(ExactSize: 4);
194 break;
195 case RISCV::SD:
196 case RISCV::SD_RV32:
197 case RISCV::FSD:
198 MemBytes = TypeSize::getFixed(ExactSize: 8);
199 break;
200 case RISCV::VS1R_V:
201 case RISCV::VS2R_V:
202 case RISCV::VS4R_V:
203 case RISCV::VS8R_V:
204 if (!MI.getOperand(i: 1).isFI())
205 return Register();
206 FrameIndex = MI.getOperand(i: 1).getIndex();
207 unsigned LMUL = *getLMULForRVVWholeLoadStore(Opcode: MI.getOpcode());
208 MemBytes = TypeSize::getScalable(MinimumSize: RISCV::RVVBytesPerBlock * LMUL);
209 return MI.getOperand(i: 0).getReg();
210 }
211
212 if (MI.getOperand(i: 1).isFI() && MI.getOperand(i: 2).isImm() &&
213 MI.getOperand(i: 2).getImm() == 0) {
214 FrameIndex = MI.getOperand(i: 1).getIndex();
215 return MI.getOperand(i: 0).getReg();
216 }
217
218 return 0;
219}
220
221bool RISCVInstrInfo::isReMaterializableImpl(
222 const MachineInstr &MI) const {
223 switch (RISCV::getRVVMCOpcode(RVVPseudoOpcode: MI.getOpcode())) {
224 case RISCV::VMV_V_X:
225 case RISCV::VFMV_V_F:
226 case RISCV::VMV_V_I:
227 case RISCV::VMV_S_X:
228 case RISCV::VFMV_S_F:
229 case RISCV::VID_V:
230 return MI.getOperand(i: 1).isUndef();
231 default:
232 return TargetInstrInfo::isReMaterializableImpl(MI);
233 }
234}
235
236static bool forwardCopyWillClobberTuple(unsigned DstReg, unsigned SrcReg,
237 unsigned NumRegs) {
238 return DstReg > SrcReg && (DstReg - SrcReg) < NumRegs;
239}
240
241static bool isConvertibleToVMV_V_V(const RISCVSubtarget &STI,
242 const MachineBasicBlock &MBB,
243 MachineBasicBlock::const_iterator MBBI,
244 MachineBasicBlock::const_iterator &DefMBBI,
245 RISCVVType::VLMUL LMul) {
246 if (STI.getCLOpts().prefer_whole_register_move)
247 return false;
248
249 assert(MBBI->getOpcode() == TargetOpcode::COPY &&
250 "Unexpected COPY instruction.");
251 Register SrcReg = MBBI->getOperand(i: 1).getReg();
252 const TargetRegisterInfo *TRI = STI.getRegisterInfo();
253
254 bool FoundDef = false;
255 bool FirstVSetVLI = false;
256 unsigned FirstSEW = 0;
257 while (MBBI != MBB.begin()) {
258 --MBBI;
259 if (MBBI->isMetaInstruction())
260 continue;
261
262 if (RISCVInstrInfo::isVectorConfigInstr(MI: *MBBI)) {
263 // There is a vsetvli between COPY and source define instruction.
264 // vy = def_vop ... (producing instruction)
265 // ...
266 // vsetvli
267 // ...
268 // vx = COPY vy
269 if (!FoundDef) {
270 if (!FirstVSetVLI) {
271 FirstVSetVLI = true;
272 unsigned FirstVType = MBBI->getOperand(i: 2).getImm();
273 RISCVVType::VLMUL FirstLMul = RISCVVType::getVLMUL(VType: FirstVType);
274 FirstSEW = RISCVVType::getSEW(VType: FirstVType);
275 // The first encountered vsetvli must have the same lmul as the
276 // register class of COPY.
277 if (FirstLMul != LMul)
278 return false;
279 }
280 // Only permit `vsetvli x0, x0, vtype` between COPY and the source
281 // define instruction.
282 if (!RISCVInstrInfo::isVLPreservingConfig(MI: *MBBI))
283 return false;
284 continue;
285 }
286
287 // MBBI is the first vsetvli before the producing instruction.
288 unsigned VType = MBBI->getOperand(i: 2).getImm();
289 // If there is a vsetvli between COPY and the producing instruction.
290 if (FirstVSetVLI) {
291 // If SEW is different, return false.
292 if (RISCVVType::getSEW(VType) != FirstSEW)
293 return false;
294 }
295
296 // If the vsetvli is tail undisturbed, keep the whole register move.
297 if (!RISCVVType::isTailAgnostic(VType))
298 return false;
299
300 // The checking is conservative. We only have register classes for
301 // LMUL = 1/2/4/8. We should be able to convert vmv1r.v to vmv.v.v
302 // for fractional LMUL operations. However, we could not use the vsetvli
303 // lmul for widening operations. The result of widening operation is
304 // 2 x LMUL.
305 return LMul == RISCVVType::getVLMUL(VType);
306 } else if (MBBI->isInlineAsm() || MBBI->isCall()) {
307 return false;
308 } else if (MBBI->getNumDefs()) {
309 // Check all the instructions which will change VL.
310 // For example, vleff has implicit def VL.
311 if (MBBI->modifiesRegister(Reg: RISCV::VL, /*TRI=*/nullptr))
312 return false;
313
314 // Only converting whole register copies to vmv.v.v when the defining
315 // value appears in the explicit operands.
316 for (const MachineOperand &MO : MBBI->explicit_operands()) {
317 if (!MO.isReg() || !MO.isDef())
318 continue;
319 if (!FoundDef && TRI->regsOverlap(RegA: MO.getReg(), RegB: SrcReg)) {
320 // We only permit the source of COPY has the same LMUL as the defined
321 // operand.
322 // There are cases we need to keep the whole register copy if the LMUL
323 // is different.
324 // For example,
325 // $x0 = PseudoVSETIVLI 4, 73 // vsetivli zero, 4, e16,m2,ta,m
326 // $v28m4 = PseudoVWADD_VV_M2 $v26m2, $v8m2
327 // # The COPY may be created by vlmul_trunc intrinsic.
328 // $v26m2 = COPY renamable $v28m2, implicit killed $v28m4
329 //
330 // After widening, the valid value will be 4 x e32 elements. If we
331 // convert the COPY to vmv.v.v, it will only copy 4 x e16 elements.
332 // FIXME: The COPY of subregister of Zvlsseg register will not be able
333 // to convert to vmv.v.[v|i] under the constraint.
334 if (MO.getReg() != SrcReg)
335 return false;
336
337 // In widening reduction instructions with LMUL_1 input vector case,
338 // only checking the LMUL is insufficient due to reduction result is
339 // always LMUL_1.
340 // For example,
341 // $x11 = PseudoVSETIVLI 1, 64 // vsetivli a1, 1, e8, m1, ta, mu
342 // $v8m1 = PseudoVWREDSUM_VS_M1 $v26, $v27
343 // $v26 = COPY killed renamable $v8
344 // After widening, The valid value will be 1 x e16 elements. If we
345 // convert the COPY to vmv.v.v, it will only copy 1 x e8 elements.
346 uint64_t TSFlags = MBBI->getDesc().TSFlags;
347 if (RISCVII::isRVVWideningReduction(TSFlags))
348 return false;
349
350 // If the producing instruction does not depend on vsetvli, do not
351 // convert COPY to vmv.v.v. For example, VL1R_V or PseudoVRELOAD.
352 if (!RISCVII::hasSEWOp(TSFlags) || !RISCVII::hasVLOp(TSFlags))
353 return false;
354
355 // Found the definition.
356 FoundDef = true;
357 DefMBBI = MBBI;
358 break;
359 }
360 }
361 }
362 }
363
364 return false;
365}
366
367void RISCVInstrInfo::copyPhysRegVector(
368 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
369 const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc,
370 const TargetRegisterClass *RegClass) const {
371 const RISCVRegisterInfo *TRI = STI.getRegisterInfo();
372 RISCVVType::VLMUL LMul = RISCVRI::getLMul(TSFlags: RegClass->TSFlags);
373 unsigned NF = RISCVRI::getNF(TSFlags: RegClass->TSFlags);
374
375 uint16_t SrcEncoding = TRI->getEncodingValue(Reg: SrcReg);
376 uint16_t DstEncoding = TRI->getEncodingValue(Reg: DstReg);
377 auto [LMulVal, Fractional] = RISCVVType::decodeVLMUL(VLMul: LMul);
378 assert(!Fractional && "It is impossible be fractional lmul here.");
379 unsigned NumRegs = NF * LMulVal;
380 bool ReversedCopy =
381 forwardCopyWillClobberTuple(DstReg: DstEncoding, SrcReg: SrcEncoding, NumRegs);
382 if (ReversedCopy) {
383 // If the src and dest overlap when copying a tuple, we need to copy the
384 // registers in reverse.
385 SrcEncoding += NumRegs - 1;
386 DstEncoding += NumRegs - 1;
387 }
388
389 unsigned I = 0;
390 auto GetCopyInfo = [&](uint16_t SrcEncoding, uint16_t DstEncoding)
391 -> std::tuple<RISCVVType::VLMUL, const TargetRegisterClass &, unsigned,
392 unsigned, unsigned> {
393 if (ReversedCopy) {
394 // For reversed copying, if there are enough aligned registers(8/4/2), we
395 // can do a larger copy(LMUL8/4/2).
396 // Besides, we have already known that DstEncoding is larger than
397 // SrcEncoding in forwardCopyWillClobberTuple, so the difference between
398 // DstEncoding and SrcEncoding should be >= LMUL value we try to use to
399 // avoid clobbering.
400 uint16_t Diff = DstEncoding - SrcEncoding;
401 if (I + 8 <= NumRegs && Diff >= 8 && SrcEncoding % 8 == 7 &&
402 DstEncoding % 8 == 7)
403 return {RISCVVType::LMUL_8, RISCV::VRM8RegClass, RISCV::VMV8R_V,
404 RISCV::PseudoVMV_V_V_M8, RISCV::PseudoVMV_V_I_M8};
405 if (I + 4 <= NumRegs && Diff >= 4 && SrcEncoding % 4 == 3 &&
406 DstEncoding % 4 == 3)
407 return {RISCVVType::LMUL_4, RISCV::VRM4RegClass, RISCV::VMV4R_V,
408 RISCV::PseudoVMV_V_V_M4, RISCV::PseudoVMV_V_I_M4};
409 if (I + 2 <= NumRegs && Diff >= 2 && SrcEncoding % 2 == 1 &&
410 DstEncoding % 2 == 1)
411 return {RISCVVType::LMUL_2, RISCV::VRM2RegClass, RISCV::VMV2R_V,
412 RISCV::PseudoVMV_V_V_M2, RISCV::PseudoVMV_V_I_M2};
413 // Or we should do LMUL1 copying.
414 return {RISCVVType::LMUL_1, RISCV::VRRegClass, RISCV::VMV1R_V,
415 RISCV::PseudoVMV_V_V_M1, RISCV::PseudoVMV_V_I_M1};
416 }
417
418 // For forward copying, if source register encoding and destination register
419 // encoding are aligned to 8/4/2, we can do a LMUL8/4/2 copying.
420 if (I + 8 <= NumRegs && SrcEncoding % 8 == 0 && DstEncoding % 8 == 0)
421 return {RISCVVType::LMUL_8, RISCV::VRM8RegClass, RISCV::VMV8R_V,
422 RISCV::PseudoVMV_V_V_M8, RISCV::PseudoVMV_V_I_M8};
423 if (I + 4 <= NumRegs && SrcEncoding % 4 == 0 && DstEncoding % 4 == 0)
424 return {RISCVVType::LMUL_4, RISCV::VRM4RegClass, RISCV::VMV4R_V,
425 RISCV::PseudoVMV_V_V_M4, RISCV::PseudoVMV_V_I_M4};
426 if (I + 2 <= NumRegs && SrcEncoding % 2 == 0 && DstEncoding % 2 == 0)
427 return {RISCVVType::LMUL_2, RISCV::VRM2RegClass, RISCV::VMV2R_V,
428 RISCV::PseudoVMV_V_V_M2, RISCV::PseudoVMV_V_I_M2};
429 // Or we should do LMUL1 copying.
430 return {RISCVVType::LMUL_1, RISCV::VRRegClass, RISCV::VMV1R_V,
431 RISCV::PseudoVMV_V_V_M1, RISCV::PseudoVMV_V_I_M1};
432 };
433
434 while (I != NumRegs) {
435 // For non-segment copying, we only do this once as the registers are always
436 // aligned.
437 // For segment copying, we may do this several times. If the registers are
438 // aligned to larger LMUL, we can eliminate some copyings.
439 auto [LMulCopied, RegClass, Opc, VVOpc, VIOpc] =
440 GetCopyInfo(SrcEncoding, DstEncoding);
441 auto [NumCopied, _] = RISCVVType::decodeVLMUL(VLMul: LMulCopied);
442
443 MachineBasicBlock::const_iterator DefMBBI;
444 if (LMul == LMulCopied &&
445 isConvertibleToVMV_V_V(STI, MBB, MBBI, DefMBBI, LMul)) {
446 Opc = VVOpc;
447 if (DefMBBI->getOpcode() == VIOpc)
448 Opc = VIOpc;
449 }
450
451 // Emit actual copying.
452 // For reversed copying, the encoding should be decreased.
453 MCRegister ActualSrcReg = TRI->findVRegWithEncoding(
454 RegClass, Encoding: ReversedCopy ? (SrcEncoding - NumCopied + 1) : SrcEncoding);
455 MCRegister ActualDstReg = TRI->findVRegWithEncoding(
456 RegClass, Encoding: ReversedCopy ? (DstEncoding - NumCopied + 1) : DstEncoding);
457
458 auto MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: Opc), DestReg: ActualDstReg);
459 bool UseVMV_V_I = RISCV::getRVVMCOpcode(RVVPseudoOpcode: Opc) == RISCV::VMV_V_I;
460 bool UseVMV = UseVMV_V_I || RISCV::getRVVMCOpcode(RVVPseudoOpcode: Opc) == RISCV::VMV_V_V;
461 if (UseVMV)
462 MIB.addReg(RegNo: ActualDstReg, Flags: RegState::Undef);
463 if (UseVMV_V_I)
464 MIB = MIB.add(MO: DefMBBI->getOperand(i: 2));
465 else
466 MIB = MIB.addReg(RegNo: ActualSrcReg, Flags: getKillRegState(B: KillSrc));
467 if (UseVMV) {
468 const MCInstrDesc &Desc = DefMBBI->getDesc();
469 MIB.add(MO: DefMBBI->getOperand(i: RISCVII::getVLOpNum(Desc))); // AVL
470 unsigned Log2SEW =
471 DefMBBI->getOperand(i: RISCVII::getSEWOpNum(Desc)).getImm();
472 MIB.addImm(Val: Log2SEW ? Log2SEW : 3); // SEW
473 MIB.addImm(Val: 0); // tu, mu
474 MIB.addReg(RegNo: RISCV::VL, Flags: RegState::Implicit);
475 MIB.addReg(RegNo: RISCV::VTYPE, Flags: RegState::Implicit);
476 }
477 // Add an implicit read of the original source to silence the verifier
478 // in the cases where some of the smaller VRs we're copying from might be
479 // undef, caused by the fact that the original, larger source VR might not
480 // be fully initialized at the time this COPY happens.
481 MIB.addReg(RegNo: SrcReg, Flags: RegState::Implicit);
482
483 // If we are copying reversely, we should decrease the encoding.
484 SrcEncoding += (ReversedCopy ? -NumCopied : NumCopied);
485 DstEncoding += (ReversedCopy ? -NumCopied : NumCopied);
486 I += NumCopied;
487 }
488}
489
490void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
491 MachineBasicBlock::iterator MBBI,
492 const DebugLoc &DL, Register DstReg,
493 Register SrcReg, bool KillSrc,
494 bool RenamableDest, bool RenamableSrc) const {
495 const TargetRegisterInfo *TRI = STI.getRegisterInfo();
496 RegState KillFlag = getKillRegState(B: KillSrc);
497
498 if (RISCV::GPRRegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
499 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::ADDI), DestReg: DstReg)
500 .addReg(RegNo: SrcReg, Flags: KillFlag | getRenamableRegState(B: RenamableSrc))
501 .addImm(Val: 0);
502 return;
503 }
504
505 // Extracting from X0_Pair may create copies from DUMMY_REG_PAIR_WITH_X0.
506 if (SrcReg == RISCV::DUMMY_REG_PAIR_WITH_X0 &&
507 RISCV::GPRRegClass.contains(Reg: DstReg)) {
508 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::ADDI), DestReg: DstReg)
509 .addReg(RegNo: RISCV::X0)
510 .addImm(Val: 0);
511 return;
512 }
513
514 if (RISCV::GPRF16RegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
515 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::PseudoMV_FPR16INX), DestReg: DstReg)
516 .addReg(RegNo: SrcReg, Flags: KillFlag | getRenamableRegState(B: RenamableSrc));
517 return;
518 }
519
520 if (RISCV::GPRF32RegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
521 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::PseudoMV_FPR32INX), DestReg: DstReg)
522 .addReg(RegNo: SrcReg, Flags: KillFlag | getRenamableRegState(B: RenamableSrc));
523 return;
524 }
525
526 if (RISCV::GPRPairRegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
527 if (!STI.is64Bit()) {
528 if (STI.hasStdExtZdinx()) {
529 // On RV32_Zdinx, FMV.D will move a pair of registers to another pair of
530 // registers, in one instruction.
531 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FSGNJ_D_IN32X), DestReg: DstReg)
532 .addReg(RegNo: SrcReg, Flags: getRenamableRegState(B: RenamableSrc))
533 .addReg(RegNo: SrcReg, Flags: KillFlag | getRenamableRegState(B: RenamableSrc));
534 return;
535 }
536
537 if (STI.hasStdExtP()) {
538 // On RV32P, `padd.dw` is a GPR Pair Add
539 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::PADD_DW), DestReg: DstReg)
540 .addReg(RegNo: RISCV::X0_Pair)
541 .addReg(RegNo: SrcReg, Flags: KillFlag | getRenamableRegState(B: RenamableSrc));
542 return;
543 }
544 }
545
546 MCRegister EvenReg = TRI->getSubReg(Reg: SrcReg, Idx: RISCV::sub_gpr_even);
547 MCRegister OddReg = TRI->getSubReg(Reg: SrcReg, Idx: RISCV::sub_gpr_odd);
548 // We need to correct the odd register of X0_Pair.
549 if (OddReg == RISCV::DUMMY_REG_PAIR_WITH_X0)
550 OddReg = RISCV::X0;
551 assert(DstReg != RISCV::X0_Pair && "Cannot write to X0_Pair");
552
553 // Emit an ADDI for both parts of GPRPair.
554 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::ADDI),
555 DestReg: TRI->getSubReg(Reg: DstReg, Idx: RISCV::sub_gpr_even))
556 .addReg(RegNo: EvenReg, Flags: KillFlag)
557 .addImm(Val: 0);
558 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::ADDI),
559 DestReg: TRI->getSubReg(Reg: DstReg, Idx: RISCV::sub_gpr_odd))
560 .addReg(RegNo: OddReg, Flags: KillFlag)
561 .addImm(Val: 0);
562 return;
563 }
564
565 // Handle copy from csr
566 if (RISCV::VCSRRegClass.contains(Reg: SrcReg) &&
567 RISCV::GPRRegClass.contains(Reg: DstReg)) {
568 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::CSRRS), DestReg: DstReg)
569 .addImm(Val: RISCVSysReg::lookupSysRegByName(Name: TRI->getName(RegNo: SrcReg))->Encoding)
570 .addReg(RegNo: RISCV::X0);
571 return;
572 }
573
574 if (RISCV::FPR16RegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
575 unsigned Opc;
576 if (STI.hasStdExtZfh()) {
577 Opc = RISCV::FSGNJ_H;
578 } else {
579 assert(STI.hasStdExtF() &&
580 (STI.hasStdExtZfhmin() || STI.hasStdExtZfbfmin()) &&
581 "Unexpected extensions");
582 // Zfhmin/Zfbfmin doesn't have FSGNJ_H, replace FSGNJ_H with FSGNJ_S.
583 DstReg = TRI->getMatchingSuperReg(Reg: DstReg, SubIdx: RISCV::sub_16,
584 RC: &RISCV::FPR32RegClass);
585 SrcReg = TRI->getMatchingSuperReg(Reg: SrcReg, SubIdx: RISCV::sub_16,
586 RC: &RISCV::FPR32RegClass);
587 Opc = RISCV::FSGNJ_S;
588 }
589 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: Opc), DestReg: DstReg)
590 .addReg(RegNo: SrcReg, Flags: KillFlag)
591 .addReg(RegNo: SrcReg, Flags: KillFlag);
592 return;
593 }
594
595 if (RISCV::FPR32RegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
596 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FSGNJ_S), DestReg: DstReg)
597 .addReg(RegNo: SrcReg, Flags: KillFlag)
598 .addReg(RegNo: SrcReg, Flags: KillFlag);
599 return;
600 }
601
602 if (RISCV::FPR64RegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
603 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FSGNJ_D), DestReg: DstReg)
604 .addReg(RegNo: SrcReg, Flags: KillFlag)
605 .addReg(RegNo: SrcReg, Flags: KillFlag);
606 return;
607 }
608
609 if (RISCV::FPR32RegClass.contains(Reg: DstReg) &&
610 RISCV::GPRRegClass.contains(Reg: SrcReg)) {
611 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FMV_W_X), DestReg: DstReg)
612 .addReg(RegNo: SrcReg, Flags: KillFlag);
613 return;
614 }
615
616 if (RISCV::GPRRegClass.contains(Reg: DstReg) &&
617 RISCV::FPR32RegClass.contains(Reg: SrcReg)) {
618 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FMV_X_W), DestReg: DstReg)
619 .addReg(RegNo: SrcReg, Flags: KillFlag);
620 return;
621 }
622
623 if (RISCV::FPR64RegClass.contains(Reg: DstReg) &&
624 RISCV::GPRRegClass.contains(Reg: SrcReg)) {
625 assert(STI.getXLen() == 64 && "Unexpected GPR size");
626 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FMV_D_X), DestReg: DstReg)
627 .addReg(RegNo: SrcReg, Flags: KillFlag);
628 return;
629 }
630
631 if (RISCV::GPRRegClass.contains(Reg: DstReg) &&
632 RISCV::FPR64RegClass.contains(Reg: SrcReg)) {
633 assert(STI.getXLen() == 64 && "Unexpected GPR size");
634 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FMV_X_D), DestReg: DstReg)
635 .addReg(RegNo: SrcReg, Flags: KillFlag);
636 return;
637 }
638
639 // VR->VR copies.
640 const TargetRegisterClass *RegClass =
641 TRI->getCommonMinimalPhysRegClass(Reg1: SrcReg, Reg2: DstReg);
642 if (RISCVRegisterInfo::isRVVRegClass(RC: RegClass)) {
643 copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RegClass);
644 return;
645 }
646
647 llvm_unreachable("Impossible reg-to-reg copy");
648}
649
650void RISCVInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
651 MachineBasicBlock::iterator I,
652 Register SrcReg, bool IsKill, int FI,
653 const TargetRegisterClass *RC,
654 Register VReg,
655 MachineInstr::MIFlag Flags) const {
656 MachineFunction *MF = MBB.getParent();
657 MachineFrameInfo &MFI = MF->getFrameInfo();
658 Align Alignment = MFI.getObjectAlign(ObjectIdx: FI);
659
660 unsigned Opcode;
661 if (RISCV::GPRRegClass.hasSubClassEq(RC)) {
662 Opcode = RegInfo.getRegSizeInBits(RC: RISCV::GPRRegClass) == 32 ? RISCV::SW
663 : RISCV::SD;
664 } else if (RISCV::GPRF16RegClass.hasSubClassEq(RC)) {
665 Opcode = RISCV::SH_INX;
666 } else if (RISCV::GPRF32RegClass.hasSubClassEq(RC)) {
667 Opcode = RISCV::SW_INX;
668 } else if (RISCV::GPRPairRegClass.hasSubClassEq(RC)) {
669 if (!STI.is64Bit() && STI.hasStdExtZilsd() &&
670 Alignment >= STI.getZilsdAlign()) {
671 Opcode = RISCV::SD_RV32;
672 } else {
673 Opcode = RISCV::PseudoRV32ZdinxSD;
674 }
675 } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) {
676 Opcode = RISCV::FSH;
677 } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) {
678 Opcode = RISCV::FSW;
679 } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) {
680 Opcode = RISCV::FSD;
681 } else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
682 Opcode = RISCV::VS1R_V;
683 } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
684 Opcode = RISCV::VS2R_V;
685 } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
686 Opcode = RISCV::VS4R_V;
687 } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
688 Opcode = RISCV::VS8R_V;
689 } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC))
690 Opcode = RISCV::PseudoVSPILL2_M1;
691 else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC))
692 Opcode = RISCV::PseudoVSPILL2_M2;
693 else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC))
694 Opcode = RISCV::PseudoVSPILL2_M4;
695 else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC))
696 Opcode = RISCV::PseudoVSPILL3_M1;
697 else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC))
698 Opcode = RISCV::PseudoVSPILL3_M2;
699 else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC))
700 Opcode = RISCV::PseudoVSPILL4_M1;
701 else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC))
702 Opcode = RISCV::PseudoVSPILL4_M2;
703 else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC))
704 Opcode = RISCV::PseudoVSPILL5_M1;
705 else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC))
706 Opcode = RISCV::PseudoVSPILL6_M1;
707 else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC))
708 Opcode = RISCV::PseudoVSPILL7_M1;
709 else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC))
710 Opcode = RISCV::PseudoVSPILL8_M1;
711 else
712 llvm_unreachable("Can't store this register to stack slot");
713
714 if (RISCVRegisterInfo::isRVVRegClass(RC)) {
715 MachineMemOperand *MMO = MF->getMachineMemOperand(
716 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), F: MachineMemOperand::MOStore,
717 Size: TypeSize::getScalable(MinimumSize: MFI.getObjectSize(ObjectIdx: FI)), BaseAlignment: Alignment);
718
719 MFI.setStackID(ObjectIdx: FI, ID: TargetStackID::ScalableVector);
720 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode))
721 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: IsKill))
722 .addFrameIndex(Idx: FI)
723 .addMemOperand(MMO)
724 .setMIFlag(Flags);
725 NumVRegSpilled += RegInfo.getRegSizeInBits(RC: *RC) / RISCV::RVVBitsPerBlock;
726 } else {
727 MachineMemOperand *MMO = MF->getMachineMemOperand(
728 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), F: MachineMemOperand::MOStore,
729 Size: MFI.getObjectSize(ObjectIdx: FI), BaseAlignment: Alignment);
730
731 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode))
732 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: IsKill))
733 .addFrameIndex(Idx: FI)
734 .addImm(Val: 0)
735 .addMemOperand(MMO)
736 .setMIFlag(Flags);
737 }
738}
739
740void RISCVInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
741 MachineBasicBlock::iterator I,
742 Register DstReg, int FI,
743 const TargetRegisterClass *RC,
744 Register VReg, unsigned SubReg,
745 MachineInstr::MIFlag Flags) const {
746 MachineFunction *MF = MBB.getParent();
747 MachineFrameInfo &MFI = MF->getFrameInfo();
748 Align Alignment = MFI.getObjectAlign(ObjectIdx: FI);
749 DebugLoc DL =
750 Flags & MachineInstr::FrameDestroy ? MBB.findDebugLoc(MBBI: I) : DebugLoc();
751
752 unsigned Opcode;
753 if (RISCV::GPRRegClass.hasSubClassEq(RC)) {
754 Opcode = RegInfo.getRegSizeInBits(RC: RISCV::GPRRegClass) == 32 ? RISCV::LW
755 : RISCV::LD;
756 } else if (RISCV::GPRF16RegClass.hasSubClassEq(RC)) {
757 Opcode = RISCV::LH_INX;
758 } else if (RISCV::GPRF32RegClass.hasSubClassEq(RC)) {
759 Opcode = RISCV::LW_INX;
760 } else if (RISCV::GPRPairRegClass.hasSubClassEq(RC)) {
761 if (!STI.is64Bit() && STI.hasStdExtZilsd() &&
762 Alignment >= STI.getZilsdAlign()) {
763 Opcode = RISCV::LD_RV32;
764 } else {
765 Opcode = RISCV::PseudoRV32ZdinxLD;
766 }
767 } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) {
768 Opcode = RISCV::FLH;
769 } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) {
770 Opcode = RISCV::FLW;
771 } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) {
772 Opcode = RISCV::FLD;
773 } else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
774 Opcode = RISCV::VL1RE8_V;
775 } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
776 Opcode = RISCV::VL2RE8_V;
777 } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
778 Opcode = RISCV::VL4RE8_V;
779 } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
780 Opcode = RISCV::VL8RE8_V;
781 } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC))
782 Opcode = RISCV::PseudoVRELOAD2_M1;
783 else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC))
784 Opcode = RISCV::PseudoVRELOAD2_M2;
785 else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC))
786 Opcode = RISCV::PseudoVRELOAD2_M4;
787 else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC))
788 Opcode = RISCV::PseudoVRELOAD3_M1;
789 else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC))
790 Opcode = RISCV::PseudoVRELOAD3_M2;
791 else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC))
792 Opcode = RISCV::PseudoVRELOAD4_M1;
793 else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC))
794 Opcode = RISCV::PseudoVRELOAD4_M2;
795 else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC))
796 Opcode = RISCV::PseudoVRELOAD5_M1;
797 else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC))
798 Opcode = RISCV::PseudoVRELOAD6_M1;
799 else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC))
800 Opcode = RISCV::PseudoVRELOAD7_M1;
801 else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC))
802 Opcode = RISCV::PseudoVRELOAD8_M1;
803 else
804 llvm_unreachable("Can't load this register from stack slot");
805
806 if (RISCVRegisterInfo::isRVVRegClass(RC)) {
807 MachineMemOperand *MMO = MF->getMachineMemOperand(
808 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), F: MachineMemOperand::MOLoad,
809 Size: TypeSize::getScalable(MinimumSize: MFI.getObjectSize(ObjectIdx: FI)), BaseAlignment: Alignment);
810
811 MFI.setStackID(ObjectIdx: FI, ID: TargetStackID::ScalableVector);
812 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode), DestReg: DstReg)
813 .addFrameIndex(Idx: FI)
814 .addMemOperand(MMO)
815 .setMIFlag(Flags);
816 NumVRegReloaded += RegInfo.getRegSizeInBits(RC: *RC) / RISCV::RVVBitsPerBlock;
817 } else {
818 MachineMemOperand *MMO = MF->getMachineMemOperand(
819 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), F: MachineMemOperand::MOLoad,
820 Size: MFI.getObjectSize(ObjectIdx: FI), BaseAlignment: Alignment);
821
822 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode), DestReg: DstReg)
823 .addFrameIndex(Idx: FI)
824 .addImm(Val: 0)
825 .addMemOperand(MMO)
826 .setMIFlag(Flags);
827 }
828}
829std::optional<unsigned> getFoldedOpcode(MachineFunction &MF, MachineInstr &MI,
830 ArrayRef<unsigned> Ops,
831 const RISCVSubtarget &ST) {
832
833 // The below optimizations narrow the load so they are only valid for little
834 // endian.
835 // TODO: Support big endian by adding an offset into the frame object?
836 if (MF.getDataLayout().isBigEndian())
837 return std::nullopt;
838
839 // Fold load from stack followed by sext.b/sext.h/sext.w/zext.b/zext.h/zext.w.
840 if (Ops.size() != 1 || Ops[0] != 1)
841 return std::nullopt;
842
843 switch (MI.getOpcode()) {
844 default:
845 if (RISCVInstrInfo::isSEXT_W(MI))
846 return RISCV::LW;
847 if (RISCVInstrInfo::isZEXT_W(MI))
848 return RISCV::LWU;
849 if (RISCVInstrInfo::isZEXT_B(MI))
850 return RISCV::LBU;
851 break;
852 case RISCV::SEXT_H:
853 return RISCV::LH;
854 case RISCV::SEXT_B:
855 return RISCV::LB;
856 case RISCV::ZEXT_H_RV32:
857 case RISCV::ZEXT_H_RV64:
858 return RISCV::LHU;
859 }
860
861 switch (RISCV::getRVVMCOpcode(RVVPseudoOpcode: MI.getOpcode())) {
862 default:
863 return std::nullopt;
864 case RISCV::VMV_X_S: {
865 unsigned Log2SEW =
866 MI.getOperand(i: RISCVII::getSEWOpNum(Desc: MI.getDesc())).getImm();
867 if (ST.getXLen() < (1U << Log2SEW))
868 return std::nullopt;
869 switch (Log2SEW) {
870 case 3:
871 return RISCV::LB;
872 case 4:
873 return RISCV::LH;
874 case 5:
875 return RISCV::LW;
876 case 6:
877 return RISCV::LD;
878 default:
879 llvm_unreachable("Unexpected SEW");
880 }
881 }
882 case RISCV::VFMV_F_S: {
883 unsigned Log2SEW =
884 MI.getOperand(i: RISCVII::getSEWOpNum(Desc: MI.getDesc())).getImm();
885 switch (Log2SEW) {
886 case 4:
887 return RISCV::FLH;
888 case 5:
889 return RISCV::FLW;
890 case 6:
891 return RISCV::FLD;
892 default:
893 llvm_unreachable("Unexpected SEW");
894 }
895 }
896 }
897}
898
899// This is the version used during InlineSpiller::spillAroundUses
900MachineInstr *
901RISCVInstrInfo::foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
902 ArrayRef<unsigned> Ops, int FrameIndex,
903 MachineInstr *&CopyMI, LiveIntervals *LIS,
904 VirtRegMap *VRM) const {
905 MachineBasicBlock::iterator InsertPt = MI;
906 std::optional<unsigned> LoadOpc = getFoldedOpcode(MF, MI, Ops, ST: STI);
907 if (!LoadOpc)
908 return nullptr;
909 Register DstReg = MI.getOperand(i: 0).getReg();
910 return BuildMI(BB&: *MI.getParent(), I: InsertPt, MIMD: MI.getDebugLoc(), MCID: get(Opcode: *LoadOpc),
911 DestReg: DstReg)
912 .addFrameIndex(Idx: FrameIndex)
913 .addImm(Val: 0);
914}
915
916static unsigned getLoadPredicatedOpcode(unsigned Opcode) {
917 switch (Opcode) {
918 case RISCV::LB:
919 return RISCV::PseudoCCLB;
920 case RISCV::LBU:
921 return RISCV::PseudoCCLBU;
922 case RISCV::LH:
923 return RISCV::PseudoCCLH;
924 case RISCV::LHU:
925 return RISCV::PseudoCCLHU;
926 case RISCV::LW:
927 return RISCV::PseudoCCLW;
928 case RISCV::LWU:
929 return RISCV::PseudoCCLWU;
930 case RISCV::LD:
931 return RISCV::PseudoCCLD;
932 case RISCV::QC_E_LB:
933 return RISCV::PseudoCCQC_E_LB;
934 case RISCV::QC_E_LBU:
935 return RISCV::PseudoCCQC_E_LBU;
936 case RISCV::QC_E_LH:
937 return RISCV::PseudoCCQC_E_LH;
938 case RISCV::QC_E_LHU:
939 return RISCV::PseudoCCQC_E_LHU;
940 case RISCV::QC_E_LW:
941 return RISCV::PseudoCCQC_E_LW;
942 default:
943 return 0;
944 }
945}
946
947MachineInstr *RISCVInstrInfo::foldMemoryOperandImpl(
948 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
949 MachineInstr &LoadMI, MachineInstr *&CopyMI, LiveIntervals *LIS,
950 VirtRegMap *VRM) const {
951 MachineBasicBlock::iterator InsertPt = MI;
952 // For now, only handle RISCV::PseudoCCMOVGPR.
953 if (MI.getOpcode() != RISCV::PseudoCCMOVGPR)
954 return nullptr;
955
956 unsigned PredOpc = getLoadPredicatedOpcode(Opcode: LoadMI.getOpcode());
957
958 if (!STI.hasShortForwardBranchILoad() || !PredOpc)
959 return nullptr;
960
961 MachineRegisterInfo &MRI = MF.getRegInfo();
962 if (Ops.size() != 1 || (Ops[0] != 1 && Ops[0] != 2))
963 return nullptr;
964
965 bool Invert = Ops[0] == 2;
966 const MachineOperand &FalseReg = MI.getOperand(i: !Invert ? 2 : 1);
967 Register DestReg = MI.getOperand(i: 0).getReg();
968 const TargetRegisterClass *PreviousClass = MRI.getRegClass(Reg: FalseReg.getReg());
969 if (!MRI.constrainRegClass(Reg: DestReg, RC: PreviousClass))
970 return nullptr;
971
972 // Create a new predicated version of DefMI.
973 MachineInstrBuilder NewMI = BuildMI(BB&: *MI.getParent(), I: InsertPt,
974 MIMD: MI.getDebugLoc(), MCID: get(Opcode: PredOpc), DestReg);
975
976 // Copy the false register.
977 NewMI.add(MO: FalseReg);
978
979 // Copy all the DefMI operands.
980 const MCInstrDesc &DefDesc = LoadMI.getDesc();
981 for (unsigned i = 1, e = DefDesc.getNumOperands(); i != e; ++i)
982 NewMI.add(MO: LoadMI.getOperand(i));
983
984 // Add branch opcode, inverting if necessary.
985 unsigned BCC = MI.getOperand(i: MI.getNumExplicitOperands() - 3).getImm();
986 if (!Invert)
987 BCC = RISCVCC::getInverseBranchOpcode(BCC);
988 NewMI.addImm(Val: BCC);
989
990 // Copy condition portion
991 NewMI.add(MOs: {MI.getOperand(i: MI.getNumExplicitOperands() - 2),
992 MI.getOperand(i: MI.getNumExplicitOperands() - 1)});
993 NewMI.cloneMemRefs(OtherMI: LoadMI);
994 return NewMI;
995}
996
997void RISCVInstrInfo::movImm(MachineBasicBlock &MBB,
998 MachineBasicBlock::iterator MBBI,
999 const DebugLoc &DL, Register DstReg, uint64_t Val,
1000 MachineInstr::MIFlag Flag, bool DstRenamable,
1001 bool DstIsDead) const {
1002 Register SrcReg = RISCV::X0;
1003
1004 // For RV32, allow a sign or unsigned 32 bit value.
1005 if (!STI.is64Bit() && !isInt<32>(x: Val)) {
1006 // If have a uimm32 it will still fit in a register so we can allow it.
1007 if (!isUInt<32>(x: Val))
1008 report_fatal_error(reason: "Should only materialize 32-bit constants for RV32");
1009
1010 // Sign extend for generateInstSeq.
1011 Val = SignExtend64<32>(x: Val);
1012 }
1013
1014 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val, STI);
1015 assert(!Seq.empty());
1016
1017 bool SrcRenamable = false;
1018 unsigned Num = 0;
1019
1020 for (const RISCVMatInt::Inst &Inst : Seq) {
1021 bool LastItem = ++Num == Seq.size();
1022 RegState DstRegState = getDeadRegState(B: DstIsDead && LastItem) |
1023 getRenamableRegState(B: DstRenamable);
1024 RegState SrcRegState = getKillRegState(B: SrcReg != RISCV::X0) |
1025 getRenamableRegState(B: SrcRenamable);
1026 switch (Inst.getOpndKind()) {
1027 case RISCVMatInt::Imm:
1028 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: Inst.getOpcode()))
1029 .addDef(RegNo: DstReg, Flags: DstRegState)
1030 .addImm(Val: Inst.getImm())
1031 .setMIFlag(Flag);
1032 break;
1033 case RISCVMatInt::RegX0:
1034 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: Inst.getOpcode()))
1035 .addDef(RegNo: DstReg, Flags: DstRegState)
1036 .addReg(RegNo: SrcReg, Flags: SrcRegState)
1037 .addReg(RegNo: RISCV::X0)
1038 .setMIFlag(Flag);
1039 break;
1040 case RISCVMatInt::RegReg:
1041 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: Inst.getOpcode()))
1042 .addDef(RegNo: DstReg, Flags: DstRegState)
1043 .addReg(RegNo: SrcReg, Flags: SrcRegState)
1044 .addReg(RegNo: SrcReg, Flags: SrcRegState)
1045 .setMIFlag(Flag);
1046 break;
1047 case RISCVMatInt::RegImm:
1048 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: Inst.getOpcode()))
1049 .addDef(RegNo: DstReg, Flags: DstRegState)
1050 .addReg(RegNo: SrcReg, Flags: SrcRegState)
1051 .addImm(Val: Inst.getImm())
1052 .setMIFlag(Flag);
1053 break;
1054 }
1055
1056 // Only the first instruction has X0 as its source.
1057 SrcReg = DstReg;
1058 SrcRenamable = DstRenamable;
1059 }
1060}
1061
1062RISCVCC::CondCode RISCVInstrInfo::getCondFromBranchOpc(unsigned Opc) {
1063 switch (Opc) {
1064 default:
1065 return RISCVCC::COND_INVALID;
1066 case RISCV::BEQ:
1067 case RISCV::BEQI:
1068 case RISCV::CV_BEQIMM:
1069 case RISCV::QC_BEQI:
1070 case RISCV::QC_E_BEQI:
1071 case RISCV::NDS_BBC:
1072 case RISCV::NDS_BEQC:
1073 return RISCVCC::COND_EQ;
1074 case RISCV::BNE:
1075 case RISCV::BNEI:
1076 case RISCV::QC_BNEI:
1077 case RISCV::QC_E_BNEI:
1078 case RISCV::CV_BNEIMM:
1079 case RISCV::NDS_BBS:
1080 case RISCV::NDS_BNEC:
1081 return RISCVCC::COND_NE;
1082 case RISCV::BLT:
1083 case RISCV::QC_BLTI:
1084 case RISCV::QC_E_BLTI:
1085 return RISCVCC::COND_LT;
1086 case RISCV::BGE:
1087 case RISCV::QC_BGEI:
1088 case RISCV::QC_E_BGEI:
1089 return RISCVCC::COND_GE;
1090 case RISCV::BLTU:
1091 case RISCV::QC_BLTUI:
1092 case RISCV::QC_E_BLTUI:
1093 return RISCVCC::COND_LTU;
1094 case RISCV::BGEU:
1095 case RISCV::QC_BGEUI:
1096 case RISCV::QC_E_BGEUI:
1097 return RISCVCC::COND_GEU;
1098 }
1099}
1100
1101bool RISCVInstrInfo::evaluateCondBranch(RISCVCC::CondCode CC, int64_t C0,
1102 int64_t C1) {
1103 switch (CC) {
1104 default:
1105 llvm_unreachable("Unexpected CC");
1106 case RISCVCC::COND_EQ:
1107 return C0 == C1;
1108 case RISCVCC::COND_NE:
1109 return C0 != C1;
1110 case RISCVCC::COND_LT:
1111 return C0 < C1;
1112 case RISCVCC::COND_GE:
1113 return C0 >= C1;
1114 case RISCVCC::COND_LTU:
1115 return (uint64_t)C0 < (uint64_t)C1;
1116 case RISCVCC::COND_GEU:
1117 return (uint64_t)C0 >= (uint64_t)C1;
1118 }
1119}
1120
1121// The contents of values added to Cond are not examined outside of
1122// RISCVInstrInfo, giving us flexibility in what to push to it. For RISCV, we
1123// push BranchOpcode, Reg1, Reg2.
1124static void parseCondBranch(MachineInstr &LastInst, MachineBasicBlock *&Target,
1125 SmallVectorImpl<MachineOperand> &Cond) {
1126 // Block ends with fall-through condbranch.
1127 assert(LastInst.getDesc().isConditionalBranch() &&
1128 "Unknown conditional branch");
1129 Target = LastInst.getOperand(i: 2).getMBB();
1130 Cond.push_back(Elt: MachineOperand::CreateImm(Val: LastInst.getOpcode()));
1131 Cond.push_back(Elt: LastInst.getOperand(i: 0));
1132 Cond.push_back(Elt: LastInst.getOperand(i: 1));
1133}
1134
1135static unsigned getInverseXqcicmOpcode(unsigned Opcode) {
1136 switch (Opcode) {
1137 default:
1138 llvm_unreachable("Unexpected Opcode");
1139 case RISCV::QC_MVEQ:
1140 return RISCV::QC_MVNE;
1141 case RISCV::QC_MVNE:
1142 return RISCV::QC_MVEQ;
1143 case RISCV::QC_MVLT:
1144 return RISCV::QC_MVGE;
1145 case RISCV::QC_MVGE:
1146 return RISCV::QC_MVLT;
1147 case RISCV::QC_MVLTU:
1148 return RISCV::QC_MVGEU;
1149 case RISCV::QC_MVGEU:
1150 return RISCV::QC_MVLTU;
1151 case RISCV::QC_MVEQI:
1152 return RISCV::QC_MVNEI;
1153 case RISCV::QC_MVNEI:
1154 return RISCV::QC_MVEQI;
1155 case RISCV::QC_MVLTI:
1156 return RISCV::QC_MVGEI;
1157 case RISCV::QC_MVGEI:
1158 return RISCV::QC_MVLTI;
1159 case RISCV::QC_MVLTUI:
1160 return RISCV::QC_MVGEUI;
1161 case RISCV::QC_MVGEUI:
1162 return RISCV::QC_MVLTUI;
1163 }
1164}
1165
1166unsigned RISCVCC::getBrCond(RISCVCC::CondCode CC, unsigned SelectOpc) {
1167 switch (SelectOpc) {
1168 default:
1169 switch (CC) {
1170 default:
1171 llvm_unreachable("Unexpected condition code!");
1172 case RISCVCC::COND_EQ:
1173 return RISCV::BEQ;
1174 case RISCVCC::COND_NE:
1175 return RISCV::BNE;
1176 case RISCVCC::COND_LT:
1177 return RISCV::BLT;
1178 case RISCVCC::COND_GE:
1179 return RISCV::BGE;
1180 case RISCVCC::COND_LTU:
1181 return RISCV::BLTU;
1182 case RISCVCC::COND_GEU:
1183 return RISCV::BGEU;
1184 }
1185 break;
1186 case RISCV::Select_GPR_Using_CC_Imm5_Zibi:
1187 switch (CC) {
1188 default:
1189 llvm_unreachable("Unexpected condition code!");
1190 case RISCVCC::COND_EQ:
1191 return RISCV::BEQI;
1192 case RISCVCC::COND_NE:
1193 return RISCV::BNEI;
1194 }
1195 break;
1196 case RISCV::Select_GPR_Using_CC_SImm5_CV:
1197 switch (CC) {
1198 default:
1199 llvm_unreachable("Unexpected condition code!");
1200 case RISCVCC::COND_EQ:
1201 return RISCV::CV_BEQIMM;
1202 case RISCVCC::COND_NE:
1203 return RISCV::CV_BNEIMM;
1204 }
1205 break;
1206 case RISCV::Select_GPRNoX0_Using_CC_SImm5NonZero_QC:
1207 switch (CC) {
1208 default:
1209 llvm_unreachable("Unexpected condition code!");
1210 case RISCVCC::COND_EQ:
1211 return RISCV::QC_BEQI;
1212 case RISCVCC::COND_NE:
1213 return RISCV::QC_BNEI;
1214 case RISCVCC::COND_LT:
1215 return RISCV::QC_BLTI;
1216 case RISCVCC::COND_GE:
1217 return RISCV::QC_BGEI;
1218 }
1219 break;
1220 case RISCV::Select_GPRNoX0_Using_CC_UImm5NonZero_QC:
1221 switch (CC) {
1222 default:
1223 llvm_unreachable("Unexpected condition code!");
1224 case RISCVCC::COND_LTU:
1225 return RISCV::QC_BLTUI;
1226 case RISCVCC::COND_GEU:
1227 return RISCV::QC_BGEUI;
1228 }
1229 break;
1230 case RISCV::Select_GPRNoX0_Using_CC_SImm16NonZero_QC:
1231 switch (CC) {
1232 default:
1233 llvm_unreachable("Unexpected condition code!");
1234 case RISCVCC::COND_EQ:
1235 return RISCV::QC_E_BEQI;
1236 case RISCVCC::COND_NE:
1237 return RISCV::QC_E_BNEI;
1238 case RISCVCC::COND_LT:
1239 return RISCV::QC_E_BLTI;
1240 case RISCVCC::COND_GE:
1241 return RISCV::QC_E_BGEI;
1242 }
1243 break;
1244 case RISCV::Select_GPRNoX0_Using_CC_UImm16NonZero_QC:
1245 switch (CC) {
1246 default:
1247 llvm_unreachable("Unexpected condition code!");
1248 case RISCVCC::COND_LTU:
1249 return RISCV::QC_E_BLTUI;
1250 case RISCVCC::COND_GEU:
1251 return RISCV::QC_E_BGEUI;
1252 }
1253 break;
1254 case RISCV::Select_GPR_Using_CC_UImmLog2XLen_NDS:
1255 switch (CC) {
1256 default:
1257 llvm_unreachable("Unexpected condition code!");
1258 case RISCVCC::COND_EQ:
1259 return RISCV::NDS_BBC;
1260 case RISCVCC::COND_NE:
1261 return RISCV::NDS_BBS;
1262 }
1263 break;
1264 case RISCV::Select_GPR_Using_CC_UImm7_NDS:
1265 switch (CC) {
1266 default:
1267 llvm_unreachable("Unexpected condition code!");
1268 case RISCVCC::COND_EQ:
1269 return RISCV::NDS_BEQC;
1270 case RISCVCC::COND_NE:
1271 return RISCV::NDS_BNEC;
1272 }
1273 break;
1274 }
1275}
1276
1277RISCVCC::CondCode RISCVCC::getInverseBranchCondition(RISCVCC::CondCode CC) {
1278 switch (CC) {
1279 default:
1280 llvm_unreachable("Unrecognized conditional branch");
1281 case RISCVCC::COND_EQ:
1282 return RISCVCC::COND_NE;
1283 case RISCVCC::COND_NE:
1284 return RISCVCC::COND_EQ;
1285 case RISCVCC::COND_LT:
1286 return RISCVCC::COND_GE;
1287 case RISCVCC::COND_GE:
1288 return RISCVCC::COND_LT;
1289 case RISCVCC::COND_LTU:
1290 return RISCVCC::COND_GEU;
1291 case RISCVCC::COND_GEU:
1292 return RISCVCC::COND_LTU;
1293 }
1294}
1295
1296// Return inverse branch
1297unsigned RISCVCC::getInverseBranchOpcode(unsigned BCC) {
1298 switch (BCC) {
1299 default:
1300 llvm_unreachable("Unexpected branch opcode!");
1301 case RISCV::BEQ:
1302 return RISCV::BNE;
1303 case RISCV::BEQI:
1304 return RISCV::BNEI;
1305 case RISCV::BNE:
1306 return RISCV::BEQ;
1307 case RISCV::BNEI:
1308 return RISCV::BEQI;
1309 case RISCV::BLT:
1310 return RISCV::BGE;
1311 case RISCV::BGE:
1312 return RISCV::BLT;
1313 case RISCV::BLTU:
1314 return RISCV::BGEU;
1315 case RISCV::BGEU:
1316 return RISCV::BLTU;
1317 case RISCV::CV_BEQIMM:
1318 return RISCV::CV_BNEIMM;
1319 case RISCV::CV_BNEIMM:
1320 return RISCV::CV_BEQIMM;
1321 case RISCV::QC_BEQI:
1322 return RISCV::QC_BNEI;
1323 case RISCV::QC_BNEI:
1324 return RISCV::QC_BEQI;
1325 case RISCV::QC_BLTI:
1326 return RISCV::QC_BGEI;
1327 case RISCV::QC_BGEI:
1328 return RISCV::QC_BLTI;
1329 case RISCV::QC_BLTUI:
1330 return RISCV::QC_BGEUI;
1331 case RISCV::QC_BGEUI:
1332 return RISCV::QC_BLTUI;
1333 case RISCV::QC_E_BEQI:
1334 return RISCV::QC_E_BNEI;
1335 case RISCV::QC_E_BNEI:
1336 return RISCV::QC_E_BEQI;
1337 case RISCV::QC_E_BLTI:
1338 return RISCV::QC_E_BGEI;
1339 case RISCV::QC_E_BGEI:
1340 return RISCV::QC_E_BLTI;
1341 case RISCV::QC_E_BLTUI:
1342 return RISCV::QC_E_BGEUI;
1343 case RISCV::QC_E_BGEUI:
1344 return RISCV::QC_E_BLTUI;
1345 case RISCV::NDS_BBC:
1346 return RISCV::NDS_BBS;
1347 case RISCV::NDS_BBS:
1348 return RISCV::NDS_BBC;
1349 case RISCV::NDS_BEQC:
1350 return RISCV::NDS_BNEC;
1351 case RISCV::NDS_BNEC:
1352 return RISCV::NDS_BEQC;
1353 }
1354}
1355
1356bool RISCVInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
1357 MachineBasicBlock *&TBB,
1358 MachineBasicBlock *&FBB,
1359 SmallVectorImpl<MachineOperand> &Cond,
1360 bool AllowModify) const {
1361 TBB = FBB = nullptr;
1362 Cond.clear();
1363
1364 // If the block has no terminators, it just falls into the block after it.
1365 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
1366 if (I == MBB.end() || !isUnpredicatedTerminator(MI: *I))
1367 return false;
1368
1369 // Count the number of terminators and find the first unconditional or
1370 // indirect branch.
1371 MachineBasicBlock::iterator FirstUncondOrIndirectBr = MBB.end();
1372 int NumTerminators = 0;
1373 for (auto J = I.getReverse(); J != MBB.rend() && isUnpredicatedTerminator(MI: *J);
1374 J++) {
1375 NumTerminators++;
1376 if (J->getDesc().isUnconditionalBranch() ||
1377 J->getDesc().isIndirectBranch()) {
1378 FirstUncondOrIndirectBr = J.getReverse();
1379 }
1380 }
1381
1382 // If AllowModify is true, we can erase any terminators after
1383 // FirstUncondOrIndirectBR.
1384 if (AllowModify && FirstUncondOrIndirectBr != MBB.end()) {
1385 while (std::next(x: FirstUncondOrIndirectBr) != MBB.end()) {
1386 std::next(x: FirstUncondOrIndirectBr)->eraseFromParent();
1387 NumTerminators--;
1388 }
1389 I = FirstUncondOrIndirectBr;
1390 }
1391
1392 // We can't handle blocks that end in an indirect branch.
1393 if (I->getDesc().isIndirectBranch())
1394 return true;
1395
1396 // We can't handle Generic branch opcodes from Global ISel.
1397 if (I->isPreISelOpcode())
1398 return true;
1399
1400 // We can't handle blocks with more than 2 terminators.
1401 if (NumTerminators > 2)
1402 return true;
1403
1404 // Handle a single unconditional branch.
1405 if (NumTerminators == 1 && I->getDesc().isUnconditionalBranch()) {
1406 TBB = getBranchDestBlock(MI: *I);
1407 return false;
1408 }
1409
1410 // Handle a single conditional branch.
1411 if (NumTerminators == 1 && I->getDesc().isConditionalBranch()) {
1412 parseCondBranch(LastInst&: *I, Target&: TBB, Cond);
1413 return false;
1414 }
1415
1416 // Handle a conditional branch followed by an unconditional branch.
1417 if (NumTerminators == 2 && std::prev(x: I)->getDesc().isConditionalBranch() &&
1418 I->getDesc().isUnconditionalBranch()) {
1419 parseCondBranch(LastInst&: *std::prev(x: I), Target&: TBB, Cond);
1420 FBB = getBranchDestBlock(MI: *I);
1421 return false;
1422 }
1423
1424 // Otherwise, we can't handle this.
1425 return true;
1426}
1427
1428unsigned RISCVInstrInfo::removeBranch(MachineBasicBlock &MBB,
1429 int *BytesRemoved) const {
1430 if (BytesRemoved)
1431 *BytesRemoved = 0;
1432 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
1433 if (I == MBB.end())
1434 return 0;
1435
1436 if (!I->getDesc().isUnconditionalBranch() &&
1437 !I->getDesc().isConditionalBranch())
1438 return 0;
1439
1440 // Remove the branch.
1441 if (BytesRemoved)
1442 *BytesRemoved += getInstSizeInBytes(MI: *I);
1443 I->eraseFromParent();
1444
1445 I = MBB.end();
1446
1447 if (I == MBB.begin())
1448 return 1;
1449 --I;
1450 if (!I->getDesc().isConditionalBranch())
1451 return 1;
1452
1453 // Remove the branch.
1454 if (BytesRemoved)
1455 *BytesRemoved += getInstSizeInBytes(MI: *I);
1456 I->eraseFromParent();
1457 return 2;
1458}
1459
1460// Inserts a branch into the end of the specific MachineBasicBlock, returning
1461// the number of instructions inserted.
1462unsigned RISCVInstrInfo::insertBranch(
1463 MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
1464 ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const {
1465 if (BytesAdded)
1466 *BytesAdded = 0;
1467
1468 // Shouldn't be a fall through.
1469 assert(TBB && "insertBranch must not be told to insert a fallthrough");
1470 assert((Cond.size() == 3 || Cond.size() == 0) &&
1471 "RISC-V branch conditions have two components!");
1472
1473 // Unconditional branch.
1474 if (Cond.empty()) {
1475 MachineInstr &MI = *BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: RISCV::PseudoBR)).addMBB(MBB: TBB);
1476 if (BytesAdded)
1477 *BytesAdded += getInstSizeInBytes(MI);
1478 return 1;
1479 }
1480
1481 // Either a one or two-way conditional branch.
1482 MachineInstr &CondMI = *BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: Cond[0].getImm()))
1483 .add(MO: Cond[1])
1484 .add(MO: Cond[2])
1485 .addMBB(MBB: TBB);
1486 if (BytesAdded)
1487 *BytesAdded += getInstSizeInBytes(MI: CondMI);
1488
1489 // One-way conditional branch.
1490 if (!FBB)
1491 return 1;
1492
1493 // Two-way conditional branch.
1494 MachineInstr &MI = *BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: RISCV::PseudoBR)).addMBB(MBB: FBB);
1495 if (BytesAdded)
1496 *BytesAdded += getInstSizeInBytes(MI);
1497 return 2;
1498}
1499
1500void RISCVInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
1501 MachineBasicBlock &DestBB,
1502 MachineBasicBlock &RestoreBB,
1503 const DebugLoc &DL, int64_t BrOffset,
1504 RegScavenger *RS) const {
1505 assert(RS && "RegScavenger required for long branching");
1506 assert(MBB.empty() &&
1507 "new block should be inserted for expanding unconditional branch");
1508 assert(MBB.pred_size() == 1);
1509 assert(RestoreBB.empty() &&
1510 "restore block should be inserted for restoring clobbered registers");
1511
1512 MachineFunction *MF = MBB.getParent();
1513 MachineRegisterInfo &MRI = MF->getRegInfo();
1514 RISCVMachineFunctionInfo *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
1515 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
1516
1517 if (!isInt<32>(x: BrOffset))
1518 report_fatal_error(
1519 reason: "Branch offsets outside of the signed 32-bit range not supported");
1520
1521 // FIXME: A virtual register must be used initially, as the register
1522 // scavenger won't work with empty blocks (SIInstrInfo::insertIndirectBranch
1523 // uses the same workaround).
1524 Register ScratchReg = MRI.createVirtualRegister(RegClass: &RISCV::GPRJALRRegClass);
1525 auto II = MBB.end();
1526 // We may also update the jump target to RestoreBB later.
1527 MachineInstr &MI = *BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::PseudoJump))
1528 .addDef(RegNo: ScratchReg, Flags: RegState::Dead)
1529 .addMBB(MBB: &DestBB, TargetFlags: RISCVII::MO_CALL);
1530
1531 RS->enterBasicBlockEnd(MBB);
1532 // When cf-protection-branch is enabled, we must use t2 (x7) for software
1533 // guarded branches to hold the landing pad label.
1534 bool HasCFBranch =
1535 MF->getInfo<RISCVMachineFunctionInfo>()->hasCFProtectionBranch();
1536 const TargetRegisterClass *RC = &RISCV::GPRRegClass;
1537 if (HasCFBranch)
1538 RC = &RISCV::GPRX7RegClass;
1539 Register TmpGPR =
1540 RS->scavengeRegisterBackwards(RC: *RC, To: MI.getIterator(),
1541 /*RestoreAfter=*/false, /*SpAdj=*/SPAdj: 0,
1542 /*AllowSpill=*/false);
1543 if (TmpGPR.isValid())
1544 RS->setRegUsed(Reg: TmpGPR);
1545 else {
1546 // The case when there is no scavenged register needs special handling.
1547
1548 // Pick s11(or s1 for rve) because it doesn't make a difference.
1549 TmpGPR = STI.hasStdExtE() ? RISCV::X9 : RISCV::X27;
1550 // Force t2 if cf-protection-branch is enabled
1551 if (HasCFBranch)
1552 TmpGPR = RISCV::X7;
1553
1554 int FrameIndex = RVFI->getBranchRelaxationScratchFrameIndex();
1555 if (FrameIndex == -1)
1556 report_fatal_error(reason: "underestimated function size");
1557
1558 storeRegToStackSlot(MBB, I: MI, SrcReg: TmpGPR, /*IsKill=*/true, FI: FrameIndex,
1559 RC: &RISCV::GPRRegClass, VReg: Register());
1560 TRI->eliminateFrameIndex(MI: std::prev(x: MI.getIterator()),
1561 /*SpAdj=*/SPAdj: 0, /*FIOperandNum=*/1);
1562
1563 MI.getOperand(i: 1).setMBB(&RestoreBB);
1564
1565 loadRegFromStackSlot(MBB&: RestoreBB, I: RestoreBB.end(), DstReg: TmpGPR, FI: FrameIndex,
1566 RC: &RISCV::GPRRegClass, VReg: Register());
1567 TRI->eliminateFrameIndex(MI: RestoreBB.back(),
1568 /*SpAdj=*/SPAdj: 0, /*FIOperandNum=*/1);
1569 }
1570
1571 MRI.replaceRegWith(FromReg: ScratchReg, ToReg: TmpGPR);
1572 MRI.clearVirtRegs();
1573}
1574
1575bool RISCVInstrInfo::reverseBranchCondition(
1576 SmallVectorImpl<MachineOperand> &Cond) const {
1577 assert((Cond.size() == 3) && "Invalid branch condition!");
1578
1579 Cond[0].setImm(RISCVCC::getInverseBranchOpcode(BCC: Cond[0].getImm()));
1580
1581 return false;
1582}
1583
1584// Return true if the instruction is a load immediate instruction (i.e.
1585// (ADDI x0, imm) or (BSETI x0, imm)).
1586static bool isLoadImm(const MachineInstr *MI, int64_t &Imm) {
1587 if (MI->getOpcode() == RISCV::ADDI && MI->getOperand(i: 1).isReg() &&
1588 MI->getOperand(i: 1).getReg() == RISCV::X0) {
1589 Imm = MI->getOperand(i: 2).getImm();
1590 return true;
1591 }
1592 // BSETI can be used to create power of 2 constants. Only 2048 is currently
1593 // interesting because it is 1 more than the maximum ADDI constant.
1594 if (MI->getOpcode() == RISCV::BSETI && MI->getOperand(i: 1).isReg() &&
1595 MI->getOperand(i: 1).getReg() == RISCV::X0 &&
1596 MI->getOperand(i: 2).getImm() == 11) {
1597 Imm = 2048;
1598 return true;
1599 }
1600 return false;
1601}
1602
1603bool RISCVInstrInfo::isFromLoadImm(const MachineRegisterInfo &MRI,
1604 const MachineOperand &Op, int64_t &Imm) {
1605 // Either a load from immediate instruction or X0.
1606 if (!Op.isReg())
1607 return false;
1608
1609 Register Reg = Op.getReg();
1610 if (Reg == RISCV::X0) {
1611 Imm = 0;
1612 return true;
1613 }
1614
1615 if (!Reg.isVirtual())
1616 return false;
1617
1618 const MachineInstr *DefMI = MRI.getVRegDef(Reg);
1619 return DefMI && isLoadImm(MI: DefMI, Imm);
1620}
1621
1622bool RISCVInstrInfo::optimizeCondBranch(MachineInstr &MI) const {
1623 bool IsSigned = false;
1624 bool IsEquality = false;
1625 switch (MI.getOpcode()) {
1626 default:
1627 return false;
1628 case RISCV::BEQ:
1629 case RISCV::BNE:
1630 IsEquality = true;
1631 break;
1632 case RISCV::BGE:
1633 case RISCV::BLT:
1634 IsSigned = true;
1635 break;
1636 case RISCV::BGEU:
1637 case RISCV::BLTU:
1638 break;
1639 }
1640
1641 MachineBasicBlock *MBB = MI.getParent();
1642 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
1643
1644 const MachineOperand &LHS = MI.getOperand(i: 0);
1645 const MachineOperand &RHS = MI.getOperand(i: 1);
1646 MachineBasicBlock *TBB = MI.getOperand(i: 2).getMBB();
1647
1648 RISCVCC::CondCode CC = getCondFromBranchOpc(Opc: MI.getOpcode());
1649 assert(CC != RISCVCC::COND_INVALID);
1650
1651 // Canonicalize conditional branches which can be constant folded into
1652 // beqz or bnez. We can't modify the CFG here.
1653 int64_t C0, C1;
1654 if (isFromLoadImm(MRI, Op: LHS, Imm&: C0) && isFromLoadImm(MRI, Op: RHS, Imm&: C1)) {
1655 unsigned NewOpc = evaluateCondBranch(CC, C0, C1) ? RISCV::BEQ : RISCV::BNE;
1656 // Build the new branch and remove the old one.
1657 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
1658 .addReg(RegNo: RISCV::X0)
1659 .addReg(RegNo: RISCV::X0)
1660 .addMBB(MBB: TBB);
1661 MI.eraseFromParent();
1662 return true;
1663 }
1664
1665 if (IsEquality)
1666 return false;
1667
1668 // For two constants C0 and C1 from
1669 // ```
1670 // li Y, C0
1671 // li Z, C1
1672 // ```
1673 // 1. if C1 = C0 + 1
1674 // we can turn:
1675 // (a) blt Y, X -> bge X, Z
1676 // (b) bge Y, X -> blt X, Z
1677 //
1678 // 2. if C1 = C0 - 1
1679 // we can turn:
1680 // (a) blt X, Y -> bge Z, X
1681 // (b) bge X, Y -> blt Z, X
1682 //
1683 // To make sure this optimization is really beneficial, we only
1684 // optimize for cases where Y had only one use (i.e. only used by the branch).
1685 // Try to find the register for constant Z; return
1686 // invalid register otherwise.
1687 auto searchConst = [&](int64_t C1) -> Register {
1688 MachineBasicBlock::reverse_iterator II(&MI), E = MBB->rend();
1689 auto DefC1 = std::find_if(first: ++II, last: E, pred: [&](const MachineInstr &I) -> bool {
1690 int64_t Imm;
1691 return isLoadImm(MI: &I, Imm) && Imm == C1 &&
1692 I.getOperand(i: 0).getReg().isVirtual();
1693 });
1694 if (DefC1 != E)
1695 return DefC1->getOperand(i: 0).getReg();
1696
1697 return Register();
1698 };
1699
1700 unsigned NewOpc = RISCVCC::getBrCond(CC: getInverseBranchCondition(CC));
1701
1702 // Might be case 1.
1703 // Don't change 0 to 1 since we can use x0.
1704 // For unsigned cases changing -1U to 0 would be incorrect.
1705 // The incorrect case for signed would be INT_MAX, but isFromLoadImm can't
1706 // return that.
1707 if (isFromLoadImm(MRI, Op: LHS, Imm&: C0) && C0 != 0 && LHS.getReg().isVirtual() &&
1708 MRI.hasOneUse(RegNo: LHS.getReg()) && (IsSigned || C0 != -1)) {
1709 assert((isInt<12>(C0) || C0 == 2048) && "Unexpected immediate");
1710 if (Register RegZ = searchConst(C0 + 1)) {
1711 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
1712 .add(MO: RHS)
1713 .addReg(RegNo: RegZ)
1714 .addMBB(MBB: TBB);
1715 // We might extend the live range of Z, clear its kill flag to
1716 // account for this.
1717 MRI.clearKillFlags(Reg: RegZ);
1718 MI.eraseFromParent();
1719 return true;
1720 }
1721 }
1722
1723 // Might be case 2.
1724 // For signed cases we don't want to change 0 since we can use x0.
1725 // For unsigned cases changing 0 to -1U would be incorrect.
1726 // The incorrect case for signed would be INT_MIN, but isFromLoadImm can't
1727 // return that.
1728 if (isFromLoadImm(MRI, Op: RHS, Imm&: C0) && C0 != 0 && RHS.getReg().isVirtual() &&
1729 MRI.hasOneUse(RegNo: RHS.getReg())) {
1730 assert((isInt<12>(C0) || C0 == 2048) && "Unexpected immediate");
1731 if (Register RegZ = searchConst(C0 - 1)) {
1732 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
1733 .addReg(RegNo: RegZ)
1734 .add(MO: LHS)
1735 .addMBB(MBB: TBB);
1736 // We might extend the live range of Z, clear its kill flag to
1737 // account for this.
1738 MRI.clearKillFlags(Reg: RegZ);
1739 MI.eraseFromParent();
1740 return true;
1741 }
1742 }
1743
1744 return false;
1745}
1746
1747MachineBasicBlock *
1748RISCVInstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
1749 assert(MI.getDesc().isBranch() && "Unexpected opcode!");
1750 // The branch target is always the last operand.
1751 int NumOp = MI.getNumExplicitOperands();
1752 return MI.getOperand(i: NumOp - 1).getMBB();
1753}
1754
1755bool RISCVInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
1756 int64_t BrOffset) const {
1757 unsigned XLen = STI.getXLen();
1758 // Ideally we could determine the supported branch offset from the
1759 // RISCVII::FormMask, but this can't be used for Pseudo instructions like
1760 // PseudoBR.
1761 switch (BranchOp) {
1762 default:
1763 llvm_unreachable("Unexpected opcode!");
1764 case RISCV::NDS_BBC:
1765 case RISCV::NDS_BBS:
1766 case RISCV::NDS_BEQC:
1767 case RISCV::NDS_BNEC:
1768 return isInt<11>(x: BrOffset);
1769 case RISCV::BEQ:
1770 case RISCV::BNE:
1771 case RISCV::BLT:
1772 case RISCV::BGE:
1773 case RISCV::BLTU:
1774 case RISCV::BGEU:
1775 case RISCV::BEQI:
1776 case RISCV::BNEI:
1777 case RISCV::CV_BEQIMM:
1778 case RISCV::CV_BNEIMM:
1779 case RISCV::QC_BEQI:
1780 case RISCV::QC_BNEI:
1781 case RISCV::QC_BGEI:
1782 case RISCV::QC_BLTI:
1783 case RISCV::QC_BLTUI:
1784 case RISCV::QC_BGEUI:
1785 case RISCV::QC_E_BEQI:
1786 case RISCV::QC_E_BNEI:
1787 case RISCV::QC_E_BGEI:
1788 case RISCV::QC_E_BLTI:
1789 case RISCV::QC_E_BLTUI:
1790 case RISCV::QC_E_BGEUI:
1791 return isInt<13>(x: BrOffset);
1792 case RISCV::JAL:
1793 case RISCV::PseudoBR:
1794 return isInt<21>(x: BrOffset);
1795 case RISCV::PseudoJump:
1796 return isInt<32>(x: SignExtend64(X: BrOffset + 0x800, B: XLen));
1797 }
1798}
1799
1800static bool isJumpTableLoad(const MachineInstr &MI) {
1801 return any_of(Range: MI.memoperands(), P: [](const MachineMemOperand *MMO) {
1802 const PseudoSourceValue *PSV = MMO->getPseudoValue();
1803 return PSV && PSV->isJumpTable();
1804 });
1805}
1806
1807// We want this instruction to be loading the base address of a jump table into
1808// a register. This can be PseudoMovAddr/PseudoLLA/LUI(+ADDI)/QC_E_LI.
1809static int getJumpTableIndexFromBase(const MachineRegisterInfo &MRI,
1810 Register Reg) {
1811 if (!Reg.isVirtual())
1812 return -1;
1813 const MachineInstr *MI = MRI.getUniqueVRegDef(Reg);
1814 if (!MI)
1815 return -1;
1816
1817 for (const MachineOperand &MO : MI->operands())
1818 if (MO.isJTI())
1819 return MO.getIndex();
1820
1821 return -1;
1822}
1823
1824// This instruction is used as the base address of a jump table load. We expect
1825// it to be adding the jump table base address to an index that may be scaled.
1826static int getJumpTableIndexFromLoadAddr(const MachineRegisterInfo &MRI,
1827 Register Reg) {
1828 if (!Reg.isVirtual())
1829 return -1;
1830 const MachineInstr *MI = MRI.getUniqueVRegDef(Reg);
1831 if (!MI)
1832 return -1;
1833
1834 int JTI;
1835 switch (MI->getOpcode()) {
1836 case RISCV::SH1ADD:
1837 case RISCV::SH2ADD:
1838 case RISCV::SH3ADD:
1839 // Only the index should be scaled so we just check the unscaled operand for
1840 // the base address.
1841 // TODO: Can the address be SHXADD_UW?
1842 JTI = getJumpTableIndexFromBase(MRI, Reg: MI->getOperand(i: 2).getReg());
1843 if (JTI >= 0)
1844 return JTI;
1845 break;
1846 case RISCV::ADD:
1847 JTI = getJumpTableIndexFromBase(MRI, Reg: MI->getOperand(i: 1).getReg());
1848 if (JTI >= 0)
1849 return JTI;
1850 JTI = getJumpTableIndexFromBase(MRI, Reg: MI->getOperand(i: 2).getReg());
1851 if (JTI >= 0)
1852 return JTI;
1853 break;
1854 }
1855
1856 return -1;
1857}
1858
1859// Recursively search for %jump-table.N starting from PseudoBRIND,
1860// and return the index of %jump-table.N.
1861//
1862// One common jump table:
1863//
1864// %base = PseudoMovAddr/PseudoLLA/LUI(+ADDI)/QC_E_LI %jump-table.N
1865// %addr = SH2ADD %index, %base
1866// %entry = LW %addr, 0 :: (load from jump-table)
1867// %target = ADD %entry, %base
1868// PseudoBRIND %target, 0
1869//
1870int RISCVInstrInfo::getJumpTableIndex(const MachineInstr &MI) const {
1871 if (MI.getOpcode() != RISCV::PseudoBRIND &&
1872 MI.getOpcode() != RISCV::PseudoBRINDX7)
1873 return -1;
1874
1875 Register Reg = MI.getOperand(i: 0).getReg();
1876 if (!Reg.isVirtual())
1877 return -1;
1878
1879 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1880 MachineInstr *Def = MRI.getUniqueVRegDef(Reg);
1881 if (!Def)
1882 return -1;
1883
1884 // The target may come directly from a load or the jump table may store
1885 // relative offset that needs the table base added to it.
1886 int JTI;
1887 switch (Def->getOpcode()) {
1888 case RISCV::LW:
1889 case RISCV::LWU:
1890 case RISCV::LD:
1891 if (!isJumpTableLoad(MI: *Def))
1892 return -1;
1893
1894 JTI = getJumpTableIndexFromLoadAddr(MRI, Reg: Def->getOperand(i: 1).getReg());
1895 if (JTI >= 0)
1896 return JTI;
1897 break;
1898 case RISCV::LXSW:
1899 case RISCV::LXWU:
1900 case RISCV::LXSD:
1901 case RISCV::QC_LRW:
1902 if (!isJumpTableLoad(MI: *Def))
1903 return -1;
1904
1905 JTI = getJumpTableIndexFromBase(MRI, Reg: Def->getOperand(i: 1).getReg());
1906 if (JTI >= 0)
1907 return JTI;
1908 break;
1909 case RISCV::ADD:
1910 JTI = getJumpTableIndexFromBase(MRI, Reg: Def->getOperand(i: 1).getReg());
1911 if (JTI >= 0)
1912 return JTI;
1913 JTI = getJumpTableIndexFromBase(MRI, Reg: Def->getOperand(i: 2).getReg());
1914 if (JTI >= 0)
1915 return JTI;
1916 break;
1917 }
1918
1919 return -1;
1920}
1921
1922// If the operation has a predicated pseudo instruction, return the pseudo
1923// instruction opcode. Otherwise, return RISCV::INSTRUCTION_LIST_END.
1924// TODO: Support more operations.
1925unsigned getPredicatedOpcode(unsigned Opcode) {
1926 // clang-format off
1927 switch (Opcode) {
1928 case RISCV::ADD: return RISCV::PseudoCCADD;
1929 case RISCV::SUB: return RISCV::PseudoCCSUB;
1930 case RISCV::SLL: return RISCV::PseudoCCSLL;
1931 case RISCV::SRL: return RISCV::PseudoCCSRL;
1932 case RISCV::SRA: return RISCV::PseudoCCSRA;
1933 case RISCV::AND: return RISCV::PseudoCCAND;
1934 case RISCV::OR: return RISCV::PseudoCCOR;
1935 case RISCV::XOR: return RISCV::PseudoCCXOR;
1936 case RISCV::MAX: return RISCV::PseudoCCMAX;
1937 case RISCV::MAXU: return RISCV::PseudoCCMAXU;
1938 case RISCV::MIN: return RISCV::PseudoCCMIN;
1939 case RISCV::MINU: return RISCV::PseudoCCMINU;
1940 case RISCV::MUL: return RISCV::PseudoCCMUL;
1941 case RISCV::LUI: return RISCV::PseudoCCLUI;
1942 case RISCV::QC_LI: return RISCV::PseudoCCQC_LI;
1943 case RISCV::QC_E_LI: return RISCV::PseudoCCQC_E_LI;
1944
1945 case RISCV::ADDI: return RISCV::PseudoCCADDI;
1946 case RISCV::SLLI: return RISCV::PseudoCCSLLI;
1947 case RISCV::SRLI: return RISCV::PseudoCCSRLI;
1948 case RISCV::SRAI: return RISCV::PseudoCCSRAI;
1949 case RISCV::ANDI: return RISCV::PseudoCCANDI;
1950 case RISCV::ORI: return RISCV::PseudoCCORI;
1951 case RISCV::XORI: return RISCV::PseudoCCXORI;
1952
1953 case RISCV::ADDW: return RISCV::PseudoCCADDW;
1954 case RISCV::SUBW: return RISCV::PseudoCCSUBW;
1955 case RISCV::SLLW: return RISCV::PseudoCCSLLW;
1956 case RISCV::SRLW: return RISCV::PseudoCCSRLW;
1957 case RISCV::SRAW: return RISCV::PseudoCCSRAW;
1958
1959 case RISCV::ADDIW: return RISCV::PseudoCCADDIW;
1960 case RISCV::SLLIW: return RISCV::PseudoCCSLLIW;
1961 case RISCV::SRLIW: return RISCV::PseudoCCSRLIW;
1962 case RISCV::SRAIW: return RISCV::PseudoCCSRAIW;
1963
1964 case RISCV::ANDN: return RISCV::PseudoCCANDN;
1965 case RISCV::ORN: return RISCV::PseudoCCORN;
1966 case RISCV::XNOR: return RISCV::PseudoCCXNOR;
1967
1968 case RISCV::NDS_BFOS: return RISCV::PseudoCCNDS_BFOS;
1969 case RISCV::NDS_BFOZ: return RISCV::PseudoCCNDS_BFOZ;
1970 }
1971 // clang-format on
1972
1973 return RISCV::INSTRUCTION_LIST_END;
1974}
1975
1976/// Identify instructions that can be folded into a CCMOV instruction, and
1977/// return the defining instruction.
1978static MachineInstr *canFoldAsPredicatedOp(Register Reg,
1979 const MachineRegisterInfo &MRI,
1980 const TargetInstrInfo *TII,
1981 const RISCVSubtarget &STI) {
1982 if (!Reg.isVirtual())
1983 return nullptr;
1984 if (!MRI.hasOneNonDBGUse(RegNo: Reg))
1985 return nullptr;
1986 MachineInstr *MI = MRI.getVRegDef(Reg);
1987 if (!MI)
1988 return nullptr;
1989
1990 if (!STI.hasShortForwardBranchIMinMax() &&
1991 (MI->getOpcode() == RISCV::MAX || MI->getOpcode() == RISCV::MIN ||
1992 MI->getOpcode() == RISCV::MINU || MI->getOpcode() == RISCV::MAXU))
1993 return nullptr;
1994
1995 if (!STI.hasShortForwardBranchIMul() && MI->getOpcode() == RISCV::MUL)
1996 return nullptr;
1997
1998 // Check if MI can be predicated and folded into the CCMOV.
1999 if (getPredicatedOpcode(Opcode: MI->getOpcode()) == RISCV::INSTRUCTION_LIST_END)
2000 return nullptr;
2001 // Don't predicate li idiom.
2002 if (MI->getOpcode() == RISCV::ADDI && MI->getOperand(i: 1).isReg() &&
2003 MI->getOperand(i: 1).getReg() == RISCV::X0)
2004 return nullptr;
2005 // Check if MI has any other defs or physreg uses.
2006 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands())) {
2007 // Reject frame index operands, PEI can't handle the predicated pseudos.
2008 if (MO.isFI() || MO.isCPI() || MO.isJTI())
2009 return nullptr;
2010 if (!MO.isReg())
2011 continue;
2012 // MI can't have any tied operands, that would conflict with predication.
2013 if (MO.isTied())
2014 return nullptr;
2015 if (MO.isDef())
2016 return nullptr;
2017 // Allow constant physregs.
2018 if (MO.getReg().isPhysical() && !MRI.isConstantPhysReg(PhysReg: MO.getReg()))
2019 return nullptr;
2020 }
2021 bool DontMoveAcrossStores = true;
2022 if (!MI->isSafeToMove(SawStore&: DontMoveAcrossStores))
2023 return nullptr;
2024 return MI;
2025}
2026
2027MachineInstr *
2028RISCVInstrInfo::optimizeSelect(MachineInstr &MI,
2029 SmallPtrSetImpl<MachineInstr *> &SeenMIs,
2030 bool PreferFalse) const {
2031 assert(MI.getOpcode() == RISCV::PseudoCCMOVGPR &&
2032 "Unknown select instruction");
2033 if (!STI.hasShortForwardBranchIALU())
2034 return nullptr;
2035
2036 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
2037 MachineInstr *DefMI =
2038 canFoldAsPredicatedOp(Reg: MI.getOperand(i: 2).getReg(), MRI, TII: this, STI);
2039 bool Invert = !DefMI;
2040 if (!DefMI)
2041 DefMI = canFoldAsPredicatedOp(Reg: MI.getOperand(i: 1).getReg(), MRI, TII: this, STI);
2042 if (!DefMI)
2043 return nullptr;
2044
2045 // Find new register class to use.
2046 MachineOperand FalseReg = MI.getOperand(i: Invert ? 2 : 1);
2047 Register DestReg = MI.getOperand(i: 0).getReg();
2048 const TargetRegisterClass *PreviousClass = MRI.getRegClass(Reg: FalseReg.getReg());
2049 if (!MRI.constrainRegClass(Reg: DestReg, RC: PreviousClass))
2050 return nullptr;
2051
2052 unsigned PredOpc = getPredicatedOpcode(Opcode: DefMI->getOpcode());
2053 assert(PredOpc != RISCV::INSTRUCTION_LIST_END && "Unexpected opcode!");
2054
2055 // Create a new predicated version of DefMI.
2056 MachineInstrBuilder NewMI =
2057 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: PredOpc), DestReg);
2058
2059 // Copy the false register.
2060 NewMI.add(MO: FalseReg);
2061
2062 // Copy all the DefMI operands.
2063 const MCInstrDesc &DefDesc = DefMI->getDesc();
2064 for (unsigned i = 1, e = DefDesc.getNumOperands(); i != e; ++i)
2065 NewMI.add(MO: DefMI->getOperand(i));
2066
2067 // Add branch opcode, inverting if necessary.
2068 unsigned BCCOpcode = MI.getOperand(i: MI.getNumExplicitOperands() - 3).getImm();
2069 if (Invert)
2070 BCCOpcode = RISCVCC::getInverseBranchOpcode(BCC: BCCOpcode);
2071 NewMI.addImm(Val: BCCOpcode);
2072
2073 // Copy the condition portion.
2074 NewMI.add(MO: MI.getOperand(i: MI.getNumExplicitOperands() - 2));
2075 NewMI.add(MO: MI.getOperand(i: MI.getNumExplicitOperands() - 1));
2076
2077 // Update SeenMIs set: register newly created MI and erase removed DefMI.
2078 SeenMIs.insert(Ptr: NewMI);
2079 SeenMIs.erase(Ptr: DefMI);
2080
2081 // If MI is inside a loop, and DefMI is outside the loop, then kill flags on
2082 // DefMI would be invalid when transferred inside the loop. Checking for a
2083 // loop is expensive, but at least remove kill flags if they are in different
2084 // BBs.
2085 if (DefMI->getParent() != MI.getParent())
2086 NewMI->clearKillInfo();
2087
2088 // The caller will erase MI, but not DefMI.
2089 DefMI->eraseFromParent();
2090 return NewMI;
2091}
2092
2093unsigned RISCVInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
2094 if (MI.isMetaInstruction())
2095 return 0;
2096
2097 unsigned Opcode = MI.getOpcode();
2098
2099 if (Opcode == TargetOpcode::INLINEASM ||
2100 Opcode == TargetOpcode::INLINEASM_BR) {
2101 const MachineFunction &MF = *MI.getParent()->getParent();
2102 return getInlineAsmLength(Str: MI.getOperand(i: 0).getSymbolName(),
2103 MAI: MF.getTarget().getMCAsmInfo());
2104 }
2105
2106 if (requiresNTLHint(MI)) {
2107 if (STI.hasStdExtZca()) {
2108 if (unsigned Size = getCompressedSize(MI, STI))
2109 return 2 + Size; // c.ntl.all + c.load/c.store
2110 return 6; // c.ntl.all + load/store
2111 }
2112 return 8; // ntl.all + load/store
2113 }
2114
2115 if (Opcode == TargetOpcode::BUNDLE)
2116 return getInstBundleSize(MI);
2117
2118 if (MI.getParent() && MI.getParent()->getParent()) {
2119 if (unsigned Size = getCompressedSize(MI, STI))
2120 return Size;
2121 }
2122
2123 switch (Opcode) {
2124 case RISCV::PseudoMV_FPR16INX:
2125 case RISCV::PseudoMV_FPR32INX:
2126 case RISCV::PseudoClearGPR:
2127 // MV is always compressible to either c.mv or c.li rd, 0.
2128 return STI.hasStdExtZca() ? 2 : 4;
2129 // Below cases are for short forward branch pseudos
2130 case RISCV::PseudoCCMOVGPRNoX0:
2131 return get(Opcode: MI.getOperand(i: MI.getNumExplicitOperands() - 3).getImm())
2132 .getSize() +
2133 2;
2134 case RISCV::PseudoCCMOVGPR:
2135 case RISCV::PseudoCCADD:
2136 case RISCV::PseudoCCSUB:
2137 case RISCV::PseudoCCSLL:
2138 case RISCV::PseudoCCSRL:
2139 case RISCV::PseudoCCSRA:
2140 case RISCV::PseudoCCAND:
2141 case RISCV::PseudoCCOR:
2142 case RISCV::PseudoCCXOR:
2143 case RISCV::PseudoCCADDI:
2144 case RISCV::PseudoCCANDI:
2145 case RISCV::PseudoCCORI:
2146 case RISCV::PseudoCCXORI:
2147 case RISCV::PseudoCCLUI:
2148 case RISCV::PseudoCCSLLI:
2149 case RISCV::PseudoCCSRLI:
2150 case RISCV::PseudoCCSRAI:
2151 case RISCV::PseudoCCADDW:
2152 case RISCV::PseudoCCSUBW:
2153 case RISCV::PseudoCCSLLW:
2154 case RISCV::PseudoCCSRLW:
2155 case RISCV::PseudoCCSRAW:
2156 case RISCV::PseudoCCADDIW:
2157 case RISCV::PseudoCCSLLIW:
2158 case RISCV::PseudoCCSRLIW:
2159 case RISCV::PseudoCCSRAIW:
2160 case RISCV::PseudoCCANDN:
2161 case RISCV::PseudoCCORN:
2162 case RISCV::PseudoCCXNOR:
2163 case RISCV::PseudoCCMAX:
2164 case RISCV::PseudoCCMIN:
2165 case RISCV::PseudoCCMAXU:
2166 case RISCV::PseudoCCMINU:
2167 case RISCV::PseudoCCMUL:
2168 case RISCV::PseudoCCLB:
2169 case RISCV::PseudoCCLH:
2170 case RISCV::PseudoCCLW:
2171 case RISCV::PseudoCCLHU:
2172 case RISCV::PseudoCCLBU:
2173 case RISCV::PseudoCCLWU:
2174 case RISCV::PseudoCCLD:
2175 case RISCV::PseudoCCQC_LI:
2176 return get(Opcode: MI.getOperand(i: MI.getNumExplicitOperands() - 3).getImm())
2177 .getSize() +
2178 4;
2179 case RISCV::PseudoCCQC_E_LI:
2180 case RISCV::PseudoCCQC_E_LB:
2181 case RISCV::PseudoCCQC_E_LH:
2182 case RISCV::PseudoCCQC_E_LW:
2183 case RISCV::PseudoCCQC_E_LHU:
2184 case RISCV::PseudoCCQC_E_LBU:
2185 return get(Opcode: MI.getOperand(i: MI.getNumExplicitOperands() - 3).getImm())
2186 .getSize() +
2187 6;
2188 case TargetOpcode::STACKMAP:
2189 // The upper bound for a stackmap intrinsic is the full length of its shadow
2190 return StackMapOpers(&MI).getNumPatchBytes();
2191 case TargetOpcode::PATCHPOINT:
2192 // The size of the patchpoint intrinsic is the number of bytes requested
2193 return PatchPointOpers(&MI).getNumPatchBytes();
2194 case TargetOpcode::STATEPOINT: {
2195 // The size of the statepoint intrinsic is the number of bytes requested
2196 unsigned NumBytes = StatepointOpers(&MI).getNumPatchBytes();
2197 // No patch bytes means at most a PseudoCall is emitted
2198 return std::max(a: NumBytes, b: 8U);
2199 }
2200 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
2201 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
2202 case TargetOpcode::PATCHABLE_TAIL_CALL: {
2203 const MachineFunction &MF = *MI.getParent()->getParent();
2204 const Function &F = MF.getFunction();
2205 if (Opcode == TargetOpcode::PATCHABLE_FUNCTION_ENTER &&
2206 F.hasFnAttribute(Kind: "patchable-function-entry")) {
2207 unsigned Num =
2208 F.getFnAttributeAsParsedInteger(Kind: "patchable-function-entry");
2209 // Number of C.NOP or NOP
2210 return (STI.hasStdExtZca() ? 2 : 4) * Num;
2211 }
2212 // XRay uses C.JAL + 21 or 33 C.NOP for each sled in RV32 and RV64,
2213 // respectively.
2214 return STI.is64Bit() ? 68 : 44;
2215 }
2216 default:
2217 return get(Opcode).getSize();
2218 }
2219}
2220
2221bool RISCVInstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const {
2222 const unsigned Opcode = MI.getOpcode();
2223 switch (Opcode) {
2224 default:
2225 break;
2226 case RISCV::FSGNJ_D:
2227 case RISCV::FSGNJ_S:
2228 case RISCV::FSGNJ_H:
2229 case RISCV::FSGNJ_D_INX:
2230 case RISCV::FSGNJ_D_IN32X:
2231 case RISCV::FSGNJ_S_INX:
2232 case RISCV::FSGNJ_H_INX:
2233 // The canonical floating-point move is fsgnj rd, rs, rs.
2234 return MI.getOperand(i: 1).isReg() && MI.getOperand(i: 2).isReg() &&
2235 MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg();
2236 case RISCV::ADDI:
2237 case RISCV::ORI:
2238 case RISCV::XORI:
2239 return (MI.getOperand(i: 1).isReg() &&
2240 MI.getOperand(i: 1).getReg() == RISCV::X0) ||
2241 (MI.getOperand(i: 2).isImm() && MI.getOperand(i: 2).getImm() == 0);
2242 }
2243 return MI.isAsCheapAsAMove();
2244}
2245
2246std::optional<DestSourcePair>
2247RISCVInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
2248 if (MI.isMoveReg())
2249 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2250 switch (MI.getOpcode()) {
2251 default:
2252 break;
2253 case RISCV::ADD:
2254 case RISCV::OR:
2255 case RISCV::XOR:
2256 if (MI.getOperand(i: 1).isReg() && MI.getOperand(i: 1).getReg() == RISCV::X0 &&
2257 MI.getOperand(i: 2).isReg())
2258 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 2)};
2259 if (MI.getOperand(i: 2).isReg() && MI.getOperand(i: 2).getReg() == RISCV::X0 &&
2260 MI.getOperand(i: 1).isReg())
2261 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2262 break;
2263 case RISCV::ADDI:
2264 // Operand 1 can be a frameindex but callers expect registers
2265 if (MI.getOperand(i: 1).isReg() && MI.getOperand(i: 2).isImm() &&
2266 MI.getOperand(i: 2).getImm() == 0)
2267 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2268 break;
2269 case RISCV::SUB:
2270 if (MI.getOperand(i: 2).isReg() && MI.getOperand(i: 2).getReg() == RISCV::X0 &&
2271 MI.getOperand(i: 1).isReg())
2272 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2273 break;
2274 case RISCV::SH1ADD:
2275 case RISCV::SH1ADD_UW:
2276 case RISCV::SH2ADD:
2277 case RISCV::SH2ADD_UW:
2278 case RISCV::SH3ADD:
2279 case RISCV::SH3ADD_UW:
2280 if (MI.getOperand(i: 1).isReg() && MI.getOperand(i: 1).getReg() == RISCV::X0 &&
2281 MI.getOperand(i: 2).isReg())
2282 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 2)};
2283 break;
2284 case RISCV::FSGNJ_D:
2285 case RISCV::FSGNJ_S:
2286 case RISCV::FSGNJ_H:
2287 case RISCV::FSGNJ_D_INX:
2288 case RISCV::FSGNJ_D_IN32X:
2289 case RISCV::FSGNJ_S_INX:
2290 case RISCV::FSGNJ_H_INX:
2291 // The canonical floating-point move is fsgnj rd, rs, rs.
2292 if (MI.getOperand(i: 1).isReg() && MI.getOperand(i: 2).isReg() &&
2293 MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg())
2294 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2295 break;
2296 }
2297 return std::nullopt;
2298}
2299
2300MachineTraceStrategy RISCVInstrInfo::getMachineCombinerTraceStrategy() const {
2301 std::optional<MachineTraceStrategy> Forced =
2302 STI.getCLOpts().force_machine_combiner_strategy;
2303 if (!Forced) {
2304 // The option is unused. Choose Local strategy only for in-order cores. When
2305 // scheduling model is unspecified, use MinInstrCount strategy as more
2306 // generic one.
2307 const auto &SchedModel = STI.getSchedModel();
2308 return (!SchedModel.hasInstrSchedModel() || SchedModel.isOutOfOrder())
2309 ? MachineTraceStrategy::TS_MinInstrCount
2310 : MachineTraceStrategy::TS_Local;
2311 }
2312 // The strategy was forced by the option.
2313 return *Forced;
2314}
2315
2316void RISCVInstrInfo::finalizeInsInstrs(
2317 MachineInstr &Root, unsigned &Pattern,
2318 SmallVectorImpl<MachineInstr *> &InsInstrs) const {
2319 int16_t FrmOpIdx =
2320 RISCV::getNamedOperandIdx(Opcode: Root.getOpcode(), Name: RISCV::OpName::frm);
2321 if (FrmOpIdx < 0) {
2322 assert(all_of(InsInstrs,
2323 [](MachineInstr *MI) {
2324 return RISCV::getNamedOperandIdx(MI->getOpcode(),
2325 RISCV::OpName::frm) < 0;
2326 }) &&
2327 "New instructions require FRM whereas the old one does not have it");
2328 return;
2329 }
2330
2331 const MachineOperand &FRM = Root.getOperand(i: FrmOpIdx);
2332 MachineFunction &MF = *Root.getMF();
2333
2334 for (auto *NewMI : InsInstrs) {
2335 // We'd already added the FRM operand.
2336 if (static_cast<unsigned>(RISCV::getNamedOperandIdx(
2337 Opcode: NewMI->getOpcode(), Name: RISCV::OpName::frm)) != NewMI->getNumOperands())
2338 continue;
2339 MachineInstrBuilder MIB(MF, NewMI);
2340 MIB.add(MO: FRM);
2341 if (FRM.getImm() == RISCVFPRndMode::DYN)
2342 MIB.addUse(RegNo: RISCV::FRM, Flags: RegState::Implicit);
2343 }
2344}
2345
2346static bool isFADD(unsigned Opc) {
2347 switch (Opc) {
2348 default:
2349 return false;
2350 case RISCV::FADD_H:
2351 case RISCV::FADD_S:
2352 case RISCV::FADD_D:
2353 return true;
2354 }
2355}
2356
2357static bool isFSUB(unsigned Opc) {
2358 switch (Opc) {
2359 default:
2360 return false;
2361 case RISCV::FSUB_H:
2362 case RISCV::FSUB_S:
2363 case RISCV::FSUB_D:
2364 return true;
2365 }
2366}
2367
2368static bool isFMUL(unsigned Opc) {
2369 switch (Opc) {
2370 default:
2371 return false;
2372 case RISCV::FMUL_H:
2373 case RISCV::FMUL_S:
2374 case RISCV::FMUL_D:
2375 return true;
2376 }
2377}
2378
2379bool RISCVInstrInfo::isVectorAssociativeAndCommutative(const MachineInstr &Inst,
2380 bool Invert) const {
2381#define OPCODE_LMUL_CASE(OPC) \
2382 case RISCV::OPC##_M1: \
2383 case RISCV::OPC##_M2: \
2384 case RISCV::OPC##_M4: \
2385 case RISCV::OPC##_M8: \
2386 case RISCV::OPC##_MF2: \
2387 case RISCV::OPC##_MF4: \
2388 case RISCV::OPC##_MF8
2389
2390#define OPCODE_LMUL_MASK_CASE(OPC) \
2391 case RISCV::OPC##_M1_MASK: \
2392 case RISCV::OPC##_M2_MASK: \
2393 case RISCV::OPC##_M4_MASK: \
2394 case RISCV::OPC##_M8_MASK: \
2395 case RISCV::OPC##_MF2_MASK: \
2396 case RISCV::OPC##_MF4_MASK: \
2397 case RISCV::OPC##_MF8_MASK
2398
2399 unsigned Opcode = Inst.getOpcode();
2400 if (Invert) {
2401 if (auto InvOpcode = getInverseOpcode(Opcode))
2402 Opcode = *InvOpcode;
2403 else
2404 return false;
2405 }
2406
2407 // clang-format off
2408 switch (Opcode) {
2409 default:
2410 return false;
2411 OPCODE_LMUL_CASE(PseudoVADD_VV):
2412 OPCODE_LMUL_MASK_CASE(PseudoVADD_VV):
2413 OPCODE_LMUL_CASE(PseudoVMUL_VV):
2414 OPCODE_LMUL_MASK_CASE(PseudoVMUL_VV):
2415 return true;
2416 }
2417 // clang-format on
2418
2419#undef OPCODE_LMUL_MASK_CASE
2420#undef OPCODE_LMUL_CASE
2421}
2422
2423bool RISCVInstrInfo::areRVVInstsReassociable(const MachineInstr &Root,
2424 const MachineInstr &Prev) const {
2425 if (!areOpcodesEqualOrInverse(Opcode1: Root.getOpcode(), Opcode2: Prev.getOpcode()))
2426 return false;
2427
2428 assert(Root.getMF() == Prev.getMF());
2429 const MachineRegisterInfo *MRI = &Root.getMF()->getRegInfo();
2430 const TargetRegisterInfo *TRI = MRI->getTargetRegisterInfo();
2431
2432 // Make sure vtype operands are also the same.
2433 const MCInstrDesc &Desc = get(Opcode: Root.getOpcode());
2434 const uint64_t TSFlags = Desc.TSFlags;
2435
2436 auto checkImmOperand = [&](unsigned OpIdx) {
2437 return Root.getOperand(i: OpIdx).getImm() == Prev.getOperand(i: OpIdx).getImm();
2438 };
2439
2440 auto checkRegOperand = [&](unsigned OpIdx) {
2441 return Root.getOperand(i: OpIdx).getReg() == Prev.getOperand(i: OpIdx).getReg();
2442 };
2443
2444 // PassThru
2445 // TODO: Potentially we can loosen the condition to consider Root to be
2446 // associable with Prev if Root has NoReg as passthru. In which case we
2447 // also need to loosen the condition on vector policies between these.
2448 if (!checkRegOperand(1))
2449 return false;
2450
2451 // SEW
2452 if (RISCVII::hasSEWOp(TSFlags) &&
2453 !checkImmOperand(RISCVII::getSEWOpNum(Desc)))
2454 return false;
2455
2456 // Mask
2457 if (RISCVII::usesMaskPolicy(TSFlags)) {
2458 const MachineBasicBlock *MBB = Root.getParent();
2459 const MachineBasicBlock::const_reverse_iterator It1(&Root);
2460 const MachineBasicBlock::const_reverse_iterator It2(&Prev);
2461 Register MI1VReg;
2462
2463 bool SeenMI2 = false;
2464 for (auto End = MBB->rend(), It = It1; It != End; ++It) {
2465 if (It == It2) {
2466 SeenMI2 = true;
2467 if (!MI1VReg.isValid())
2468 // There is no V0 def between Root and Prev; they're sharing the
2469 // same V0.
2470 break;
2471 }
2472
2473 if (It->modifiesRegister(Reg: RISCV::V0, TRI)) {
2474 Register SrcReg = It->getOperand(i: 1).getReg();
2475 // If it's not VReg it'll be more difficult to track its defs, so
2476 // bailing out here just to be safe.
2477 if (!SrcReg.isVirtual())
2478 return false;
2479
2480 if (!MI1VReg.isValid()) {
2481 // This is the V0 def for Root.
2482 MI1VReg = SrcReg;
2483 continue;
2484 }
2485
2486 // Some random mask updates.
2487 if (!SeenMI2)
2488 continue;
2489
2490 // This is the V0 def for Prev; check if it's the same as that of
2491 // Root.
2492 if (MI1VReg != SrcReg)
2493 return false;
2494 else
2495 break;
2496 }
2497 }
2498
2499 // If we haven't encountered Prev, it's likely that this function was
2500 // called in a wrong way (e.g. Root is before Prev).
2501 assert(SeenMI2 && "Prev is expected to appear before Root");
2502 }
2503
2504 // Tail / Mask policies
2505 if (RISCVII::hasVecPolicyOp(TSFlags) &&
2506 !checkImmOperand(RISCVII::getVecPolicyOpNum(Desc)))
2507 return false;
2508
2509 // VL
2510 if (RISCVII::hasVLOp(TSFlags)) {
2511 unsigned OpIdx = RISCVII::getVLOpNum(Desc);
2512 const MachineOperand &Op1 = Root.getOperand(i: OpIdx);
2513 const MachineOperand &Op2 = Prev.getOperand(i: OpIdx);
2514 if (Op1.getType() != Op2.getType())
2515 return false;
2516 switch (Op1.getType()) {
2517 case MachineOperand::MO_Register:
2518 if (Op1.getReg() != Op2.getReg())
2519 return false;
2520 break;
2521 case MachineOperand::MO_Immediate:
2522 if (Op1.getImm() != Op2.getImm())
2523 return false;
2524 break;
2525 default:
2526 llvm_unreachable("Unrecognized VL operand type");
2527 }
2528 }
2529
2530 // Rounding modes
2531 if (int Idx = RISCVII::getFRMOpNum(Desc); Idx >= 0 && !checkImmOperand(Idx))
2532 return false;
2533 if (int Idx = RISCVII::getVXRMOpNum(Desc); Idx >= 0 && !checkImmOperand(Idx))
2534 return false;
2535
2536 return true;
2537}
2538
2539// Most of our RVV pseudos have passthru operand, so the real operands
2540// start from index = 2.
2541bool RISCVInstrInfo::hasReassociableVectorSibling(const MachineInstr &Inst,
2542 bool &Commuted) const {
2543 const MachineBasicBlock *MBB = Inst.getParent();
2544 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
2545 assert(RISCVII::isFirstDefTiedToFirstUse(get(Inst.getOpcode())) &&
2546 "Expect the present of passthrough operand.");
2547 MachineInstr *MI1 = MRI.getUniqueVRegDef(Reg: Inst.getOperand(i: 2).getReg());
2548 MachineInstr *MI2 = MRI.getUniqueVRegDef(Reg: Inst.getOperand(i: 3).getReg());
2549
2550 // If only one operand has the same or inverse opcode and it's the second
2551 // source operand, the operands must be commuted.
2552 Commuted = !areRVVInstsReassociable(Root: Inst, Prev: *MI1) &&
2553 areRVVInstsReassociable(Root: Inst, Prev: *MI2);
2554 if (Commuted)
2555 std::swap(a&: MI1, b&: MI2);
2556
2557 return areRVVInstsReassociable(Root: Inst, Prev: *MI1) &&
2558 (isVectorAssociativeAndCommutative(Inst: *MI1) ||
2559 isVectorAssociativeAndCommutative(Inst: *MI1, /* Invert */ true)) &&
2560 hasReassociableOperands(Inst: *MI1, MBB) &&
2561 MRI.hasOneNonDBGUse(RegNo: MI1->getOperand(i: 0).getReg());
2562}
2563
2564bool RISCVInstrInfo::hasReassociableOperands(
2565 const MachineInstr &Inst, const MachineBasicBlock *MBB) const {
2566 if (!isVectorAssociativeAndCommutative(Inst) &&
2567 !isVectorAssociativeAndCommutative(Inst, /*Invert=*/true))
2568 return TargetInstrInfo::hasReassociableOperands(Inst, MBB);
2569
2570 const MachineOperand &Op1 = Inst.getOperand(i: 2);
2571 const MachineOperand &Op2 = Inst.getOperand(i: 3);
2572 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
2573
2574 // We need virtual register definitions for the operands that we will
2575 // reassociate.
2576 MachineInstr *MI1 = nullptr;
2577 MachineInstr *MI2 = nullptr;
2578 if (Op1.isReg() && Op1.getReg().isVirtual())
2579 MI1 = MRI.getUniqueVRegDef(Reg: Op1.getReg());
2580 if (Op2.isReg() && Op2.getReg().isVirtual())
2581 MI2 = MRI.getUniqueVRegDef(Reg: Op2.getReg());
2582
2583 // And at least one operand must be defined in MBB.
2584 return MI1 && MI2 && (MI1->getParent() == MBB || MI2->getParent() == MBB);
2585}
2586
2587void RISCVInstrInfo::getReassociateOperandIndices(
2588 const MachineInstr &Root, unsigned Pattern,
2589 std::array<unsigned, 5> &OperandIndices) const {
2590 TargetInstrInfo::getReassociateOperandIndices(Root, Pattern, OperandIndices);
2591 if (RISCV::getRVVMCOpcode(RVVPseudoOpcode: Root.getOpcode())) {
2592 // Skip the passthrough operand, so increment all indices by one.
2593 for (unsigned I = 0; I < 5; ++I)
2594 ++OperandIndices[I];
2595 }
2596}
2597
2598bool RISCVInstrInfo::hasReassociableSibling(const MachineInstr &Inst,
2599 bool &Commuted) const {
2600 if (isVectorAssociativeAndCommutative(Inst) ||
2601 isVectorAssociativeAndCommutative(Inst, /*Invert=*/true))
2602 return hasReassociableVectorSibling(Inst, Commuted);
2603
2604 if (!TargetInstrInfo::hasReassociableSibling(Inst, Commuted))
2605 return false;
2606
2607 const MachineRegisterInfo &MRI = Inst.getMF()->getRegInfo();
2608 unsigned OperandIdx = Commuted ? 2 : 1;
2609 const MachineInstr &Sibling =
2610 *MRI.getVRegDef(Reg: Inst.getOperand(i: OperandIdx).getReg());
2611
2612 int16_t InstFrmOpIdx =
2613 RISCV::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name: RISCV::OpName::frm);
2614 int16_t SiblingFrmOpIdx =
2615 RISCV::getNamedOperandIdx(Opcode: Sibling.getOpcode(), Name: RISCV::OpName::frm);
2616
2617 return (InstFrmOpIdx < 0 && SiblingFrmOpIdx < 0) ||
2618 RISCV::hasEqualFRM(MI1: Inst, MI2: Sibling);
2619}
2620
2621bool RISCVInstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst,
2622 bool Invert) const {
2623 if (isVectorAssociativeAndCommutative(Inst, Invert))
2624 return true;
2625
2626 unsigned Opc = Inst.getOpcode();
2627 if (Invert) {
2628 auto InverseOpcode = getInverseOpcode(Opcode: Opc);
2629 if (!InverseOpcode)
2630 return false;
2631 Opc = *InverseOpcode;
2632 }
2633
2634 if (isFADD(Opc) || isFMUL(Opc))
2635 return Inst.getFlag(Flag: MachineInstr::MIFlag::FmReassoc) &&
2636 Inst.getFlag(Flag: MachineInstr::MIFlag::FmNsz);
2637
2638 switch (Opc) {
2639 default:
2640 return false;
2641 case RISCV::ADD:
2642 case RISCV::ADDW:
2643 case RISCV::AND:
2644 case RISCV::OR:
2645 case RISCV::XOR:
2646 // From RISC-V ISA spec, if both the high and low bits of the same product
2647 // are required, then the recommended code sequence is:
2648 //
2649 // MULH[[S]U] rdh, rs1, rs2
2650 // MUL rdl, rs1, rs2
2651 // (source register specifiers must be in same order and rdh cannot be the
2652 // same as rs1 or rs2)
2653 //
2654 // Microarchitectures can then fuse these into a single multiply operation
2655 // instead of performing two separate multiplies.
2656 // MachineCombiner may reassociate MUL operands and lose the fusion
2657 // opportunity.
2658 case RISCV::MUL:
2659 case RISCV::MULW:
2660 case RISCV::MIN:
2661 case RISCV::MINU:
2662 case RISCV::MAX:
2663 case RISCV::MAXU:
2664 case RISCV::FMIN_H:
2665 case RISCV::FMIN_S:
2666 case RISCV::FMIN_D:
2667 case RISCV::FMAX_H:
2668 case RISCV::FMAX_S:
2669 case RISCV::FMAX_D:
2670 return true;
2671 }
2672
2673 return false;
2674}
2675
2676std::optional<unsigned>
2677RISCVInstrInfo::getInverseOpcode(unsigned Opcode) const {
2678#define RVV_OPC_LMUL_CASE(OPC, INV) \
2679 case RISCV::OPC##_M1: \
2680 return RISCV::INV##_M1; \
2681 case RISCV::OPC##_M2: \
2682 return RISCV::INV##_M2; \
2683 case RISCV::OPC##_M4: \
2684 return RISCV::INV##_M4; \
2685 case RISCV::OPC##_M8: \
2686 return RISCV::INV##_M8; \
2687 case RISCV::OPC##_MF2: \
2688 return RISCV::INV##_MF2; \
2689 case RISCV::OPC##_MF4: \
2690 return RISCV::INV##_MF4; \
2691 case RISCV::OPC##_MF8: \
2692 return RISCV::INV##_MF8
2693
2694#define RVV_OPC_LMUL_MASK_CASE(OPC, INV) \
2695 case RISCV::OPC##_M1_MASK: \
2696 return RISCV::INV##_M1_MASK; \
2697 case RISCV::OPC##_M2_MASK: \
2698 return RISCV::INV##_M2_MASK; \
2699 case RISCV::OPC##_M4_MASK: \
2700 return RISCV::INV##_M4_MASK; \
2701 case RISCV::OPC##_M8_MASK: \
2702 return RISCV::INV##_M8_MASK; \
2703 case RISCV::OPC##_MF2_MASK: \
2704 return RISCV::INV##_MF2_MASK; \
2705 case RISCV::OPC##_MF4_MASK: \
2706 return RISCV::INV##_MF4_MASK; \
2707 case RISCV::OPC##_MF8_MASK: \
2708 return RISCV::INV##_MF8_MASK
2709
2710 switch (Opcode) {
2711 default:
2712 return std::nullopt;
2713 case RISCV::FADD_H:
2714 return RISCV::FSUB_H;
2715 case RISCV::FADD_S:
2716 return RISCV::FSUB_S;
2717 case RISCV::FADD_D:
2718 return RISCV::FSUB_D;
2719 case RISCV::FSUB_H:
2720 return RISCV::FADD_H;
2721 case RISCV::FSUB_S:
2722 return RISCV::FADD_S;
2723 case RISCV::FSUB_D:
2724 return RISCV::FADD_D;
2725 case RISCV::ADD:
2726 return RISCV::SUB;
2727 case RISCV::SUB:
2728 return RISCV::ADD;
2729 case RISCV::ADDW:
2730 return RISCV::SUBW;
2731 case RISCV::SUBW:
2732 return RISCV::ADDW;
2733 // clang-format off
2734 RVV_OPC_LMUL_CASE(PseudoVADD_VV, PseudoVSUB_VV);
2735 RVV_OPC_LMUL_MASK_CASE(PseudoVADD_VV, PseudoVSUB_VV);
2736 RVV_OPC_LMUL_CASE(PseudoVSUB_VV, PseudoVADD_VV);
2737 RVV_OPC_LMUL_MASK_CASE(PseudoVSUB_VV, PseudoVADD_VV);
2738 // clang-format on
2739 }
2740
2741#undef RVV_OPC_LMUL_MASK_CASE
2742#undef RVV_OPC_LMUL_CASE
2743}
2744
2745static bool canCombineFPFusedMultiply(const MachineInstr &Root,
2746 const MachineOperand &MO,
2747 bool DoRegPressureReduce) {
2748 if (!MO.isReg() || !MO.getReg().isVirtual())
2749 return false;
2750 const MachineRegisterInfo &MRI = Root.getMF()->getRegInfo();
2751 MachineInstr *MI = MRI.getVRegDef(Reg: MO.getReg());
2752 if (!MI || !isFMUL(Opc: MI->getOpcode()))
2753 return false;
2754
2755 if (!Root.getFlag(Flag: MachineInstr::MIFlag::FmContract) ||
2756 !MI->getFlag(Flag: MachineInstr::MIFlag::FmContract))
2757 return false;
2758
2759 // Try combining even if fmul has more than one use as it eliminates
2760 // dependency between fadd(fsub) and fmul. However, it can extend liveranges
2761 // for fmul operands, so reject the transformation in register pressure
2762 // reduction mode.
2763 if (DoRegPressureReduce && !MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()))
2764 return false;
2765
2766 // Do not combine instructions from different basic blocks.
2767 if (Root.getParent() != MI->getParent())
2768 return false;
2769 return RISCV::hasEqualFRM(MI1: Root, MI2: *MI);
2770}
2771
2772static bool getFPFusedMultiplyPatterns(MachineInstr &Root,
2773 SmallVectorImpl<unsigned> &Patterns,
2774 bool DoRegPressureReduce) {
2775 unsigned Opc = Root.getOpcode();
2776 bool IsFAdd = isFADD(Opc);
2777 if (!IsFAdd && !isFSUB(Opc))
2778 return false;
2779 bool Added = false;
2780 if (canCombineFPFusedMultiply(Root, MO: Root.getOperand(i: 1),
2781 DoRegPressureReduce)) {
2782 Patterns.push_back(Elt: IsFAdd ? RISCVMachineCombinerPattern::FMADD_AX
2783 : RISCVMachineCombinerPattern::FMSUB);
2784 Added = true;
2785 }
2786 if (canCombineFPFusedMultiply(Root, MO: Root.getOperand(i: 2),
2787 DoRegPressureReduce)) {
2788 Patterns.push_back(Elt: IsFAdd ? RISCVMachineCombinerPattern::FMADD_XA
2789 : RISCVMachineCombinerPattern::FNMSUB);
2790 Added = true;
2791 }
2792 return Added;
2793}
2794
2795static bool getFPPatterns(MachineInstr &Root,
2796 SmallVectorImpl<unsigned> &Patterns,
2797 bool DoRegPressureReduce) {
2798 return getFPFusedMultiplyPatterns(Root, Patterns, DoRegPressureReduce);
2799}
2800
2801/// Utility routine that checks if \param MO is defined by an
2802/// \param CombineOpc instruction in the basic block \param MBB
2803static const MachineInstr *canCombine(const MachineBasicBlock &MBB,
2804 const MachineOperand &MO,
2805 unsigned CombineOpc) {
2806 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2807 const MachineInstr *MI = nullptr;
2808
2809 if (MO.isReg() && MO.getReg().isVirtual())
2810 MI = MRI.getUniqueVRegDef(Reg: MO.getReg());
2811 // And it needs to be in the trace (otherwise, it won't have a depth).
2812 if (!MI || MI->getParent() != &MBB || MI->getOpcode() != CombineOpc)
2813 return nullptr;
2814 // Must only used by the user we combine with.
2815 if (!MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()))
2816 return nullptr;
2817
2818 return MI;
2819}
2820
2821/// Utility routine that checks if \param MO is defined by a SLLI in \param
2822/// MBB that can be combined by splitting across 2 SHXADD instructions. The
2823/// first SHXADD shift amount is given by \param OuterShiftAmt.
2824static bool canCombineShiftIntoShXAdd(const MachineBasicBlock &MBB,
2825 const MachineOperand &MO,
2826 unsigned OuterShiftAmt) {
2827 const MachineInstr *ShiftMI = canCombine(MBB, MO, CombineOpc: RISCV::SLLI);
2828 if (!ShiftMI)
2829 return false;
2830
2831 unsigned InnerShiftAmt = ShiftMI->getOperand(i: 2).getImm();
2832 if (InnerShiftAmt < OuterShiftAmt || (InnerShiftAmt - OuterShiftAmt) > 3)
2833 return false;
2834
2835 return true;
2836}
2837
2838// Returns the shift amount from a SHXADD instruction. Returns 0 if the
2839// instruction is not a SHXADD.
2840static unsigned getSHXADDShiftAmount(unsigned Opc) {
2841 switch (Opc) {
2842 default:
2843 return 0;
2844 case RISCV::SH1ADD:
2845 return 1;
2846 case RISCV::SH2ADD:
2847 return 2;
2848 case RISCV::SH3ADD:
2849 return 3;
2850 }
2851}
2852
2853// Returns the shift amount from a SHXADD.UW instruction. Returns 0 if the
2854// instruction is not a SHXADD.UW.
2855static unsigned getSHXADDUWShiftAmount(unsigned Opc) {
2856 switch (Opc) {
2857 default:
2858 return 0;
2859 case RISCV::SH1ADD_UW:
2860 return 1;
2861 case RISCV::SH2ADD_UW:
2862 return 2;
2863 case RISCV::SH3ADD_UW:
2864 return 3;
2865 }
2866}
2867
2868// Look for opportunities to combine (sh3add Z, (add X, (slli Y, 5))) into
2869// (sh3add (sh2add Y, Z), X).
2870static bool getSHXADDPatterns(const MachineInstr &Root,
2871 SmallVectorImpl<unsigned> &Patterns) {
2872 unsigned ShiftAmt = getSHXADDShiftAmount(Opc: Root.getOpcode());
2873 if (!ShiftAmt)
2874 return false;
2875
2876 const MachineBasicBlock &MBB = *Root.getParent();
2877
2878 const MachineInstr *AddMI = canCombine(MBB, MO: Root.getOperand(i: 2), CombineOpc: RISCV::ADD);
2879 if (!AddMI)
2880 return false;
2881
2882 bool Found = false;
2883 if (canCombineShiftIntoShXAdd(MBB, MO: AddMI->getOperand(i: 1), OuterShiftAmt: ShiftAmt)) {
2884 Patterns.push_back(Elt: RISCVMachineCombinerPattern::SHXADD_ADD_SLLI_OP1);
2885 Found = true;
2886 }
2887 if (canCombineShiftIntoShXAdd(MBB, MO: AddMI->getOperand(i: 2), OuterShiftAmt: ShiftAmt)) {
2888 Patterns.push_back(Elt: RISCVMachineCombinerPattern::SHXADD_ADD_SLLI_OP2);
2889 Found = true;
2890 }
2891
2892 return Found;
2893}
2894
2895CombinerObjective RISCVInstrInfo::getCombinerObjective(unsigned Pattern) const {
2896 switch (Pattern) {
2897 case RISCVMachineCombinerPattern::FMADD_AX:
2898 case RISCVMachineCombinerPattern::FMADD_XA:
2899 case RISCVMachineCombinerPattern::FMSUB:
2900 case RISCVMachineCombinerPattern::FNMSUB:
2901 return CombinerObjective::MustReduceDepth;
2902 default:
2903 return TargetInstrInfo::getCombinerObjective(Pattern);
2904 }
2905}
2906
2907bool RISCVInstrInfo::getMachineCombinerPatterns(
2908 MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns,
2909 bool DoRegPressureReduce) const {
2910
2911 if (getFPPatterns(Root, Patterns, DoRegPressureReduce))
2912 return true;
2913
2914 if (getSHXADDPatterns(Root, Patterns))
2915 return true;
2916
2917 return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
2918 DoRegPressureReduce);
2919}
2920
2921static unsigned getFPFusedMultiplyOpcode(unsigned RootOpc, unsigned Pattern) {
2922 switch (RootOpc) {
2923 default:
2924 llvm_unreachable("Unexpected opcode");
2925 case RISCV::FADD_H:
2926 return RISCV::FMADD_H;
2927 case RISCV::FADD_S:
2928 return RISCV::FMADD_S;
2929 case RISCV::FADD_D:
2930 return RISCV::FMADD_D;
2931 case RISCV::FSUB_H:
2932 return Pattern == RISCVMachineCombinerPattern::FMSUB ? RISCV::FMSUB_H
2933 : RISCV::FNMSUB_H;
2934 case RISCV::FSUB_S:
2935 return Pattern == RISCVMachineCombinerPattern::FMSUB ? RISCV::FMSUB_S
2936 : RISCV::FNMSUB_S;
2937 case RISCV::FSUB_D:
2938 return Pattern == RISCVMachineCombinerPattern::FMSUB ? RISCV::FMSUB_D
2939 : RISCV::FNMSUB_D;
2940 }
2941}
2942
2943static unsigned getAddendOperandIdx(unsigned Pattern) {
2944 switch (Pattern) {
2945 default:
2946 llvm_unreachable("Unexpected pattern");
2947 case RISCVMachineCombinerPattern::FMADD_AX:
2948 case RISCVMachineCombinerPattern::FMSUB:
2949 return 2;
2950 case RISCVMachineCombinerPattern::FMADD_XA:
2951 case RISCVMachineCombinerPattern::FNMSUB:
2952 return 1;
2953 }
2954}
2955
2956static void combineFPFusedMultiply(MachineInstr &Root, MachineInstr &Prev,
2957 unsigned Pattern,
2958 SmallVectorImpl<MachineInstr *> &InsInstrs,
2959 SmallVectorImpl<MachineInstr *> &DelInstrs) {
2960 MachineFunction *MF = Root.getMF();
2961 MachineRegisterInfo &MRI = MF->getRegInfo();
2962 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
2963
2964 MachineOperand &Mul1 = Prev.getOperand(i: 1);
2965 MachineOperand &Mul2 = Prev.getOperand(i: 2);
2966 MachineOperand &Dst = Root.getOperand(i: 0);
2967 MachineOperand &Addend = Root.getOperand(i: getAddendOperandIdx(Pattern));
2968
2969 Register DstReg = Dst.getReg();
2970 unsigned FusedOpc = getFPFusedMultiplyOpcode(RootOpc: Root.getOpcode(), Pattern);
2971 uint32_t IntersectedFlags = Root.getFlags() & Prev.getFlags();
2972 DebugLoc MergedLoc =
2973 DILocation::getMergedLocation(LocA: Root.getDebugLoc(), LocB: Prev.getDebugLoc());
2974
2975 bool Mul1IsKill = Mul1.isKill();
2976 bool Mul2IsKill = Mul2.isKill();
2977 bool AddendIsKill = Addend.isKill();
2978
2979 // We need to clear kill flags since we may be extending the live range past
2980 // a kill. If the mul had kill flags, we can preserve those since we know
2981 // where the previous range stopped.
2982 MRI.clearKillFlags(Reg: Mul1.getReg());
2983 MRI.clearKillFlags(Reg: Mul2.getReg());
2984
2985 MachineInstrBuilder MIB =
2986 BuildMI(MF&: *MF, MIMD: MergedLoc, MCID: TII->get(Opcode: FusedOpc), DestReg: DstReg)
2987 .addReg(RegNo: Mul1.getReg(), Flags: getKillRegState(B: Mul1IsKill))
2988 .addReg(RegNo: Mul2.getReg(), Flags: getKillRegState(B: Mul2IsKill))
2989 .addReg(RegNo: Addend.getReg(), Flags: getKillRegState(B: AddendIsKill))
2990 .setMIFlags(IntersectedFlags);
2991
2992 InsInstrs.push_back(Elt: MIB);
2993 if (MRI.hasOneNonDBGUse(RegNo: Prev.getOperand(i: 0).getReg()))
2994 DelInstrs.push_back(Elt: &Prev);
2995 DelInstrs.push_back(Elt: &Root);
2996}
2997
2998// Combine patterns like (sh3add Z, (add X, (slli Y, 5))) to
2999// (sh3add (sh2add Y, Z), X) if the shift amount can be split across two
3000// shXadd instructions. The outer shXadd keeps its original opcode.
3001static void
3002genShXAddAddShift(MachineInstr &Root, unsigned AddOpIdx,
3003 SmallVectorImpl<MachineInstr *> &InsInstrs,
3004 SmallVectorImpl<MachineInstr *> &DelInstrs,
3005 DenseMap<Register, unsigned> &InstrIdxForVirtReg) {
3006 MachineFunction *MF = Root.getMF();
3007 MachineRegisterInfo &MRI = MF->getRegInfo();
3008 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
3009
3010 unsigned OuterShiftAmt = getSHXADDShiftAmount(Opc: Root.getOpcode());
3011 assert(OuterShiftAmt != 0 && "Unexpected opcode");
3012
3013 MachineInstr *AddMI = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: 2).getReg());
3014 MachineInstr *ShiftMI =
3015 MRI.getUniqueVRegDef(Reg: AddMI->getOperand(i: AddOpIdx).getReg());
3016
3017 unsigned InnerShiftAmt = ShiftMI->getOperand(i: 2).getImm();
3018 assert(InnerShiftAmt >= OuterShiftAmt && "Unexpected shift amount");
3019
3020 unsigned InnerOpc;
3021 switch (InnerShiftAmt - OuterShiftAmt) {
3022 default:
3023 llvm_unreachable("Unexpected shift amount");
3024 case 0:
3025 InnerOpc = RISCV::ADD;
3026 break;
3027 case 1:
3028 InnerOpc = RISCV::SH1ADD;
3029 break;
3030 case 2:
3031 InnerOpc = RISCV::SH2ADD;
3032 break;
3033 case 3:
3034 InnerOpc = RISCV::SH3ADD;
3035 break;
3036 }
3037
3038 const MachineOperand &X = AddMI->getOperand(i: 3 - AddOpIdx);
3039 const MachineOperand &Y = ShiftMI->getOperand(i: 1);
3040 const MachineOperand &Z = Root.getOperand(i: 1);
3041
3042 Register NewVR = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
3043
3044 auto MIB1 = BuildMI(MF&: *MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: InnerOpc), DestReg: NewVR)
3045 .addReg(RegNo: Y.getReg(), Flags: getKillRegState(B: Y.isKill()))
3046 .addReg(RegNo: Z.getReg(), Flags: getKillRegState(B: Z.isKill()));
3047 auto MIB2 = BuildMI(MF&: *MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: Root.getOpcode()),
3048 DestReg: Root.getOperand(i: 0).getReg())
3049 .addReg(RegNo: NewVR, Flags: RegState::Kill)
3050 .addReg(RegNo: X.getReg(), Flags: getKillRegState(B: X.isKill()));
3051
3052 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
3053 InsInstrs.push_back(Elt: MIB1);
3054 InsInstrs.push_back(Elt: MIB2);
3055 DelInstrs.push_back(Elt: ShiftMI);
3056 DelInstrs.push_back(Elt: AddMI);
3057 DelInstrs.push_back(Elt: &Root);
3058}
3059
3060void RISCVInstrInfo::genAlternativeCodeSequence(
3061 MachineInstr &Root, unsigned Pattern,
3062 SmallVectorImpl<MachineInstr *> &InsInstrs,
3063 SmallVectorImpl<MachineInstr *> &DelInstrs,
3064 DenseMap<Register, unsigned> &InstrIdxForVirtReg) const {
3065 MachineRegisterInfo &MRI = Root.getMF()->getRegInfo();
3066 switch (Pattern) {
3067 default:
3068 TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs,
3069 DelInstrs, InstIdxForVirtReg&: InstrIdxForVirtReg);
3070 return;
3071 case RISCVMachineCombinerPattern::FMADD_AX:
3072 case RISCVMachineCombinerPattern::FMSUB: {
3073 MachineInstr &Prev = *MRI.getVRegDef(Reg: Root.getOperand(i: 1).getReg());
3074 combineFPFusedMultiply(Root, Prev, Pattern, InsInstrs, DelInstrs);
3075 return;
3076 }
3077 case RISCVMachineCombinerPattern::FMADD_XA:
3078 case RISCVMachineCombinerPattern::FNMSUB: {
3079 MachineInstr &Prev = *MRI.getVRegDef(Reg: Root.getOperand(i: 2).getReg());
3080 combineFPFusedMultiply(Root, Prev, Pattern, InsInstrs, DelInstrs);
3081 return;
3082 }
3083 case RISCVMachineCombinerPattern::SHXADD_ADD_SLLI_OP1:
3084 genShXAddAddShift(Root, AddOpIdx: 1, InsInstrs, DelInstrs, InstrIdxForVirtReg);
3085 return;
3086 case RISCVMachineCombinerPattern::SHXADD_ADD_SLLI_OP2:
3087 genShXAddAddShift(Root, AddOpIdx: 2, InsInstrs, DelInstrs, InstrIdxForVirtReg);
3088 return;
3089 }
3090}
3091
3092bool RISCVInstrInfo::verifyInstruction(const MachineInstr &MI,
3093 StringRef &ErrInfo) const {
3094 MCInstrDesc const &Desc = MI.getDesc();
3095
3096 for (const auto &[Index, Operand] : enumerate(First: Desc.operands())) {
3097 const MachineOperand &MO = MI.getOperand(i: Index);
3098 unsigned OpType = Operand.OperandType;
3099 switch (OpType) {
3100 default:
3101 if (OpType >= RISCVOp::OPERAND_FIRST_RISCV_IMM &&
3102 OpType <= RISCVOp::OPERAND_LAST_RISCV_IMM) {
3103 if (!MO.isImm()) {
3104 ErrInfo = "Expected an immediate operand.";
3105 return false;
3106 }
3107 int64_t Imm = MO.getImm();
3108 bool Ok;
3109 switch (OpType) {
3110 default:
3111 llvm_unreachable("Unexpected operand type");
3112
3113#define CASE_OPERAND_UIMM(NUM) \
3114 case RISCVOp::OPERAND_UIMM##NUM: \
3115 Ok = isUInt<NUM>(Imm); \
3116 break;
3117#define CASE_OPERAND_UIMM_LSB_ZEROS(BITS, SUFFIX) \
3118 case RISCVOp::OPERAND_UIMM##BITS##_LSB##SUFFIX: { \
3119 constexpr size_t NumZeros = sizeof(#SUFFIX) - 1; \
3120 Ok = isShiftedUInt<BITS - NumZeros, NumZeros>(Imm); \
3121 break; \
3122 }
3123#define CASE_OPERAND_SIMM(NUM) \
3124 case RISCVOp::OPERAND_SIMM##NUM: \
3125 Ok = isInt<NUM>(Imm); \
3126 break;
3127 // clang-format off
3128 CASE_OPERAND_UIMM(1)
3129 CASE_OPERAND_UIMM(2)
3130 CASE_OPERAND_UIMM(3)
3131 CASE_OPERAND_UIMM(4)
3132 CASE_OPERAND_UIMM(5)
3133 CASE_OPERAND_UIMM(6)
3134 CASE_OPERAND_UIMM(7)
3135 CASE_OPERAND_UIMM(8)
3136 CASE_OPERAND_UIMM(9)
3137 CASE_OPERAND_UIMM(10)
3138 CASE_OPERAND_UIMM(12)
3139 CASE_OPERAND_UIMM(16)
3140 CASE_OPERAND_UIMM(32)
3141 CASE_OPERAND_UIMM(48)
3142 CASE_OPERAND_UIMM(64)
3143 CASE_OPERAND_UIMM_LSB_ZEROS(2, 0)
3144 CASE_OPERAND_UIMM_LSB_ZEROS(5, 0)
3145 CASE_OPERAND_UIMM_LSB_ZEROS(6, 0)
3146 CASE_OPERAND_UIMM_LSB_ZEROS(6, 000)
3147 CASE_OPERAND_UIMM_LSB_ZEROS(7, 00)
3148 CASE_OPERAND_UIMM_LSB_ZEROS(7, 000)
3149 CASE_OPERAND_UIMM_LSB_ZEROS(8, 00)
3150 CASE_OPERAND_UIMM_LSB_ZEROS(8, 000)
3151 CASE_OPERAND_UIMM_LSB_ZEROS(9, 000)
3152 // clang-format on
3153 case RISCVOp::OPERAND_UIMM4_PLUS1:
3154 Ok = Imm >= 1 && Imm <= 16;
3155 break;
3156 case RISCVOp::OPERAND_UIMM5_NONZERO:
3157 Ok = isUInt<5>(x: Imm) && (Imm != 0);
3158 break;
3159 case RISCVOp::OPERAND_UIMM5_GT3:
3160 Ok = isUInt<5>(x: Imm) && (Imm > 3);
3161 break;
3162 case RISCVOp::OPERAND_UIMM5_PLUS1:
3163 Ok = Imm >= 1 && Imm <= 32;
3164 break;
3165 case RISCVOp::OPERAND_UIMM6_PLUS1:
3166 Ok = Imm >= 1 && Imm <= 64;
3167 break;
3168 case RISCVOp::OPERAND_UIMM8_GE32:
3169 Ok = isUInt<8>(x: Imm) && Imm >= 32;
3170 break;
3171 case RISCVOp::OPERAND_UIMM9_YBNDSWI:
3172 Ok = RISCV::isValidYBNDSWImm(Imm);
3173 break;
3174 case RISCVOp::OPERAND_SIMM10_LSB0000_NONZERO:
3175 Ok = isShiftedInt<6, 4>(x: Imm) && (Imm != 0);
3176 break;
3177 case RISCVOp::OPERAND_UIMM10_LSB00_NONZERO:
3178 Ok = isShiftedUInt<8, 2>(x: Imm) && (Imm != 0);
3179 break;
3180 case RISCVOp::OPERAND_UIMM16_NONZERO:
3181 Ok = isUInt<16>(x: Imm) && (Imm != 0);
3182 break;
3183 case RISCVOp::OPERAND_THREE:
3184 Ok = Imm == 3;
3185 break;
3186 case RISCVOp::OPERAND_FOUR:
3187 Ok = Imm == 4;
3188 break;
3189 case RISCVOp::OPERAND_IMM5_ZIBI:
3190 Ok = (isUInt<5>(x: Imm) && Imm != 0) || Imm == -1;
3191 break;
3192 // clang-format off
3193 CASE_OPERAND_SIMM(5)
3194 CASE_OPERAND_SIMM(6)
3195 CASE_OPERAND_SIMM(8)
3196 CASE_OPERAND_SIMM(10)
3197 CASE_OPERAND_SIMM(11)
3198 CASE_OPERAND_SIMM(12)
3199 CASE_OPERAND_SIMM(26)
3200 // clang-format on
3201 case RISCVOp::OPERAND_SIMM5_PLUS1:
3202 Ok = Imm >= -15 && Imm <= 16;
3203 break;
3204 case RISCVOp::OPERAND_SIMM5_NONZERO:
3205 Ok = isInt<5>(x: Imm) && (Imm != 0);
3206 break;
3207 case RISCVOp::OPERAND_SIMM6_NONZERO:
3208 Ok = Imm != 0 && isInt<6>(x: Imm);
3209 break;
3210 case RISCVOp::OPERAND_VTYPEI10:
3211 Ok = isUInt<10>(x: Imm) && RISCVVType::isValidVType(VType: Imm);
3212 break;
3213 case RISCVOp::OPERAND_VTYPEI11:
3214 Ok = isUInt<11>(x: Imm) && RISCVVType::isValidVType(VType: Imm);
3215 break;
3216 case RISCVOp::OPERAND_SIMM12_LSB00000:
3217 Ok = isShiftedInt<7, 5>(x: Imm);
3218 break;
3219 case RISCVOp::OPERAND_SIMM16_NONZERO:
3220 Ok = isInt<16>(x: Imm) && (Imm != 0);
3221 break;
3222 case RISCVOp::OPERAND_SIMM20_LI:
3223 Ok = isInt<20>(x: Imm);
3224 break;
3225 case RISCVOp::OPERAND_UIMMLOG2XLEN:
3226 Ok = STI.is64Bit() ? isUInt<6>(x: Imm) : isUInt<5>(x: Imm);
3227 break;
3228 case RISCVOp::OPERAND_UIMMLOG2XLEN_NONZERO:
3229 Ok = STI.is64Bit() ? isUInt<6>(x: Imm) : isUInt<5>(x: Imm);
3230 Ok = Ok && Imm != 0;
3231 break;
3232 case RISCVOp::OPERAND_CLUI_IMM:
3233 Ok = (isUInt<5>(x: Imm) && Imm != 0) || (Imm >= 0xfffe0 && Imm <= 0xfffff);
3234 break;
3235 case RISCVOp::OPERAND_RVKRNUM:
3236 Ok = Imm >= 0 && Imm <= 10;
3237 break;
3238 case RISCVOp::OPERAND_RVKRNUM_0_7:
3239 Ok = Imm >= 0 && Imm <= 7;
3240 break;
3241 case RISCVOp::OPERAND_RVKRNUM_1_10:
3242 Ok = Imm >= 1 && Imm <= 10;
3243 break;
3244 case RISCVOp::OPERAND_RVKRNUM_2_14:
3245 Ok = Imm >= 2 && Imm <= 14;
3246 break;
3247 case RISCVOp::OPERAND_RLIST:
3248 Ok = Imm >= RISCVZC::RA && Imm <= RISCVZC::RA_S0_S11;
3249 break;
3250 case RISCVOp::OPERAND_RLIST_S0:
3251 Ok = Imm >= RISCVZC::RA_S0 && Imm <= RISCVZC::RA_S0_S11;
3252 break;
3253 case RISCVOp::OPERAND_STACKADJ:
3254 Ok = Imm >= 0 && Imm <= 48 && Imm % 16 == 0;
3255 break;
3256 case RISCVOp::OPERAND_FRMARG:
3257 Ok = RISCVFPRndMode::isValidRoundingMode(Mode: Imm);
3258 break;
3259 case RISCVOp::OPERAND_RTZARG:
3260 Ok = Imm == RISCVFPRndMode::RTZ;
3261 break;
3262 case RISCVOp::OPERAND_SMTVType:
3263 Ok = XSMTVTypeMode::isValidSMTVTypeMode(Mode: Imm);
3264 break;
3265 case RISCVOp::OPERAND_SMTI8:
3266 Ok = Imm == XSMTVTypeMode::SMT_I8;
3267 break;
3268 case RISCVOp::OPERAND_COND_CODE:
3269 Ok = Imm >= 0 && Imm < RISCVCC::COND_INVALID;
3270 break;
3271 case RISCVOp::OPERAND_ATOMIC_ORDERING:
3272 Ok = isValidAtomicOrdering(I: Imm);
3273 break;
3274 case RISCVOp::OPERAND_VEC_POLICY:
3275 Ok = (Imm & (RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC)) ==
3276 Imm;
3277 break;
3278 case RISCVOp::OPERAND_SEW:
3279 Ok = (isUInt<5>(x: Imm) && RISCVVType::isValidSEW(SEW: 1 << Imm));
3280 break;
3281 case RISCVOp::OPERAND_SEW_MASK:
3282 Ok = Imm == 0;
3283 break;
3284 case RISCVOp::OPERAND_VEC_RM:
3285 assert(RISCVII::hasRoundModeOp(Desc.TSFlags));
3286 if (RISCVII::usesVXRM(TSFlags: Desc.TSFlags))
3287 Ok = isUInt<2>(x: Imm);
3288 else
3289 Ok = RISCVFPRndMode::isValidRoundingMode(Mode: Imm);
3290 break;
3291 case RISCVOp::OPERAND_XSFMM_VTYPE:
3292 Ok = RISCVVType::isValidXSfmmVType(VTypeI: Imm);
3293 break;
3294 case RISCVOp::OPERAND_XSFMM_TWIDEN:
3295 Ok = Imm == 1 || Imm == 2 || Imm == 4;
3296 break;
3297 }
3298 if (!Ok) {
3299 ErrInfo = "Invalid immediate";
3300 return false;
3301 }
3302 }
3303 break;
3304 case RISCVOp::OPERAND_SIMM12_LO:
3305 // TODO: We could be stricter about what non-register operands are
3306 // allowed.
3307 if (MO.isReg()) {
3308 ErrInfo = "Expected a non-register operand.";
3309 return false;
3310 }
3311 if (MO.isImm() && !isInt<12>(x: MO.getImm())) {
3312 ErrInfo = "Invalid immediate";
3313 return false;
3314 }
3315 break;
3316 case RISCVOp::OPERAND_UIMM20_LUI:
3317 case RISCVOp::OPERAND_UIMM20_AUIPC:
3318 // TODO: We could be stricter about what non-register operands are
3319 // allowed.
3320 if (MO.isReg()) {
3321 ErrInfo = "Expected a non-register operand.";
3322 return false;
3323 }
3324 if (MO.isImm() && !isUInt<20>(x: MO.getImm())) {
3325 ErrInfo = "Invalid immediate";
3326 return false;
3327 }
3328 break;
3329 case RISCVOp::OPERAND_BARE_SIMM32:
3330 // TODO: We could be stricter about what non-register operands are
3331 // allowed.
3332 if (MO.isReg()) {
3333 ErrInfo = "Expected a non-register operand.";
3334 return false;
3335 }
3336 if (MO.isImm() && !isInt<32>(x: MO.getImm())) {
3337 ErrInfo = "Invalid immediate";
3338 return false;
3339 }
3340 break;
3341 case RISCVOp::OPERAND_AVL:
3342 if (MO.isImm()) {
3343 int64_t Imm = MO.getImm();
3344 // VLMAX is represented as -1.
3345 if (!isUInt<5>(x: Imm) && Imm != -1) {
3346 ErrInfo = "Invalid immediate";
3347 return false;
3348 }
3349 } else if (!MO.isReg()) {
3350 ErrInfo = "Expected a register or immediate operand.";
3351 return false;
3352 }
3353 break;
3354 case RISCVOp::OPERAND_SFB_RHS:
3355 if (!MO.isReg() && !MO.isImm()) {
3356 ErrInfo = "Expected a register or immediate operand.";
3357 return false;
3358 }
3359 break;
3360 }
3361 }
3362
3363 const uint64_t TSFlags = Desc.TSFlags;
3364 if (RISCVII::hasVLOp(TSFlags)) {
3365 const MachineOperand &Op = MI.getOperand(i: RISCVII::getVLOpNum(Desc));
3366 if (!Op.isImm() && !Op.isReg()) {
3367 ErrInfo = "Invalid operand type for VL operand";
3368 return false;
3369 }
3370 if (Op.isReg() && Op.getReg().isValid()) {
3371 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
3372 auto *RC = MRI.getRegClass(Reg: Op.getReg());
3373 if (!RISCV::GPRNoX0RegClass.hasSubClassEq(RC)) {
3374 ErrInfo = "Invalid register class for VL operand";
3375 return false;
3376 }
3377 }
3378 if (!RISCVII::hasSEWOp(TSFlags)) {
3379 ErrInfo = "VL operand w/o SEW operand?";
3380 return false;
3381 }
3382 }
3383 if (RISCVII::hasSEWOp(TSFlags)) {
3384 unsigned OpIdx = RISCVII::getSEWOpNum(Desc);
3385 if (!MI.getOperand(i: OpIdx).isImm()) {
3386 ErrInfo = "SEW value expected to be an immediate";
3387 return false;
3388 }
3389 uint64_t Log2SEW = MI.getOperand(i: OpIdx).getImm();
3390 if (Log2SEW > 31) {
3391 ErrInfo = "Unexpected SEW value";
3392 return false;
3393 }
3394 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
3395 if (!RISCVVType::isValidSEW(SEW)) {
3396 ErrInfo = "Unexpected SEW value";
3397 return false;
3398 }
3399 }
3400 if (RISCVII::hasVecPolicyOp(TSFlags)) {
3401 unsigned OpIdx = RISCVII::getVecPolicyOpNum(Desc);
3402 if (!MI.getOperand(i: OpIdx).isImm()) {
3403 ErrInfo = "Policy operand expected to be an immediate";
3404 return false;
3405 }
3406 uint64_t Policy = MI.getOperand(i: OpIdx).getImm();
3407 if (Policy > (RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC)) {
3408 ErrInfo = "Invalid Policy Value";
3409 return false;
3410 }
3411 if (!RISCVII::hasVLOp(TSFlags)) {
3412 ErrInfo = "policy operand w/o VL operand?";
3413 return false;
3414 }
3415
3416 // VecPolicy operands can only exist on instructions with passthru/merge
3417 // arguments. Note that not all arguments with passthru have vec policy
3418 // operands- some instructions have implicit policies.
3419 unsigned UseOpIdx;
3420 if (!MI.isRegTiedToUseOperand(DefOpIdx: 0, UseOpIdx: &UseOpIdx)) {
3421 ErrInfo = "policy operand w/o tied operand?";
3422 return false;
3423 }
3424 }
3425
3426 if (int Idx = RISCVII::getFRMOpNum(Desc);
3427 Idx >= 0 && MI.getOperand(i: Idx).getImm() == RISCVFPRndMode::DYN &&
3428 !MI.readsRegister(Reg: RISCV::FRM, /*TRI=*/nullptr)) {
3429 ErrInfo = "dynamic rounding mode should read FRM";
3430 return false;
3431 }
3432
3433 return true;
3434}
3435
3436bool RISCVInstrInfo::canFoldIntoAddrMode(const MachineInstr &MemI, Register Reg,
3437 const MachineInstr &AddrI,
3438 ExtAddrMode &AM) const {
3439 switch (MemI.getOpcode()) {
3440 default:
3441 return false;
3442 case RISCV::LB:
3443 case RISCV::LBU:
3444 case RISCV::LH:
3445 case RISCV::LH_INX:
3446 case RISCV::LHU:
3447 case RISCV::LW:
3448 case RISCV::LW_INX:
3449 case RISCV::LWU:
3450 case RISCV::LD:
3451 case RISCV::LD_RV32:
3452 case RISCV::FLH:
3453 case RISCV::FLW:
3454 case RISCV::FLD:
3455 case RISCV::SB:
3456 case RISCV::SH:
3457 case RISCV::SH_INX:
3458 case RISCV::SW:
3459 case RISCV::SW_INX:
3460 case RISCV::SD:
3461 case RISCV::SD_RV32:
3462 case RISCV::FSH:
3463 case RISCV::FSW:
3464 case RISCV::FSD:
3465 break;
3466 }
3467
3468 if (MemI.getOperand(i: 0).getReg() == Reg)
3469 return false;
3470
3471 if (AddrI.getOpcode() != RISCV::ADDI || !AddrI.getOperand(i: 1).isReg() ||
3472 !AddrI.getOperand(i: 2).isImm())
3473 return false;
3474
3475 int64_t OldOffset = MemI.getOperand(i: 2).getImm();
3476 int64_t Disp = AddrI.getOperand(i: 2).getImm();
3477 int64_t NewOffset = OldOffset + Disp;
3478 if (!STI.is64Bit())
3479 NewOffset = SignExtend64<32>(x: NewOffset);
3480
3481 if (!isInt<12>(x: NewOffset))
3482 return false;
3483
3484 AM.BaseReg = AddrI.getOperand(i: 1).getReg();
3485 AM.ScaledReg = 0;
3486 AM.Scale = 0;
3487 AM.Displacement = NewOffset;
3488 AM.Form = ExtAddrMode::Formula::Basic;
3489 return true;
3490}
3491
3492MachineInstr *RISCVInstrInfo::emitLdStWithAddr(MachineInstr &MemI,
3493 const ExtAddrMode &AM) const {
3494
3495 const DebugLoc &DL = MemI.getDebugLoc();
3496 MachineBasicBlock &MBB = *MemI.getParent();
3497
3498 assert(AM.ScaledReg == 0 && AM.Scale == 0 &&
3499 "Addressing mode not supported for folding");
3500
3501 return BuildMI(BB&: MBB, I&: MemI, MIMD: DL, MCID: get(Opcode: MemI.getOpcode()))
3502 .addReg(RegNo: MemI.getOperand(i: 0).getReg(), Flags: getDefRegState(B: MemI.mayLoad()))
3503 .addReg(RegNo: AM.BaseReg)
3504 .addImm(Val: AM.Displacement)
3505 .setMemRefs(MemI.memoperands())
3506 .setMIFlags(MemI.getFlags());
3507}
3508
3509// TODO: At the moment, MIPS introduced paring of instructions operating with
3510// word or double word. This should be extended with more instructions when more
3511// vendors support load/store pairing.
3512bool RISCVInstrInfo::isPairableLdStInstOpc(unsigned Opc) {
3513 switch (Opc) {
3514 default:
3515 return false;
3516 case RISCV::SW:
3517 case RISCV::SD:
3518 case RISCV::LD:
3519 case RISCV::LW:
3520 return true;
3521 }
3522}
3523
3524bool RISCVInstrInfo::isLdStSafeToPair(const MachineInstr &LdSt,
3525 const TargetRegisterInfo *TRI) {
3526 // If this is a volatile load/store, don't mess with it.
3527 if (LdSt.hasOrderedMemoryRef() || LdSt.getNumExplicitOperands() != 3)
3528 return false;
3529
3530 if (LdSt.getOperand(i: 1).isFI())
3531 return true;
3532
3533 assert(LdSt.getOperand(1).isReg() && "Expected a reg operand.");
3534 // Can't cluster if the instruction modifies the base register
3535 // or it is update form. e.g. ld x5,8(x5)
3536 if (LdSt.modifiesRegister(Reg: LdSt.getOperand(i: 1).getReg(), TRI))
3537 return false;
3538
3539 if (!LdSt.getOperand(i: 2).isImm())
3540 return false;
3541
3542 return true;
3543}
3544
3545bool RISCVInstrInfo::getMemOperandsWithOffsetWidth(
3546 const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
3547 int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width) const {
3548 if (!LdSt.mayLoadOrStore())
3549 return false;
3550
3551 // Conservatively, only handle scalar loads/stores for now.
3552 switch (LdSt.getOpcode()) {
3553 case RISCV::LB:
3554 case RISCV::LBU:
3555 case RISCV::SB:
3556 case RISCV::LH:
3557 case RISCV::LH_INX:
3558 case RISCV::LHU:
3559 case RISCV::FLH:
3560 case RISCV::SH:
3561 case RISCV::SH_INX:
3562 case RISCV::FSH:
3563 case RISCV::LW:
3564 case RISCV::LW_INX:
3565 case RISCV::LWU:
3566 case RISCV::FLW:
3567 case RISCV::SW:
3568 case RISCV::SW_INX:
3569 case RISCV::FSW:
3570 case RISCV::LD:
3571 case RISCV::LD_RV32:
3572 case RISCV::FLD:
3573 case RISCV::SD:
3574 case RISCV::SD_RV32:
3575 case RISCV::FSD:
3576 break;
3577 default:
3578 return false;
3579 }
3580 const MachineOperand *BaseOp;
3581 OffsetIsScalable = false;
3582 if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width))
3583 return false;
3584 BaseOps.push_back(Elt: BaseOp);
3585 return true;
3586}
3587
3588// TODO: This was copied from SIInstrInfo. Could it be lifted to a common
3589// helper?
3590static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
3591 ArrayRef<const MachineOperand *> BaseOps1,
3592 const MachineInstr &MI2,
3593 ArrayRef<const MachineOperand *> BaseOps2) {
3594 // Only examine the first "base" operand of each instruction, on the
3595 // assumption that it represents the real base address of the memory access.
3596 // Other operands are typically offsets or indices from this base address.
3597 if (BaseOps1.front()->isIdenticalTo(Other: *BaseOps2.front()))
3598 return true;
3599
3600 if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
3601 return false;
3602
3603 auto MO1 = *MI1.memoperands_begin();
3604 auto MO2 = *MI2.memoperands_begin();
3605 if (MO1->getAddrSpace() != MO2->getAddrSpace())
3606 return false;
3607
3608 auto Base1 = MO1->getValue();
3609 auto Base2 = MO2->getValue();
3610 if (!Base1 || !Base2)
3611 return false;
3612 Base1 = getUnderlyingObject(V: Base1);
3613 Base2 = getUnderlyingObject(V: Base2);
3614
3615 if (isa<UndefValue>(Val: Base1) || isa<UndefValue>(Val: Base2))
3616 return false;
3617
3618 return Base1 == Base2;
3619}
3620
3621bool RISCVInstrInfo::shouldClusterMemOps(
3622 ArrayRef<const MachineOperand *> BaseOps1, int64_t Offset1,
3623 bool OffsetIsScalable1, ArrayRef<const MachineOperand *> BaseOps2,
3624 int64_t Offset2, bool OffsetIsScalable2, unsigned ClusterSize,
3625 unsigned NumBytes) const {
3626 // If the mem ops (to be clustered) do not have the same base ptr, then they
3627 // should not be clustered
3628 if (!BaseOps1.empty() && !BaseOps2.empty()) {
3629 const MachineInstr &FirstLdSt = *BaseOps1.front()->getParent();
3630 const MachineInstr &SecondLdSt = *BaseOps2.front()->getParent();
3631 if (!memOpsHaveSameBasePtr(MI1: FirstLdSt, BaseOps1, MI2: SecondLdSt, BaseOps2))
3632 return false;
3633 } else if (!BaseOps1.empty() || !BaseOps2.empty()) {
3634 // If only one base op is empty, they do not have the same base ptr
3635 return false;
3636 }
3637
3638 unsigned CacheLineSize =
3639 BaseOps1.front()->getParent()->getMF()->getSubtarget().getCacheLineSize();
3640 // Assume a cache line size of 64 bytes if no size is set in RISCVSubtarget.
3641 CacheLineSize = CacheLineSize ? CacheLineSize : 64;
3642 // Cluster if the memory operations are on the same or a neighbouring cache
3643 // line, but limit the maximum ClusterSize to avoid creating too much
3644 // additional register pressure.
3645 return ClusterSize <= 4 && std::abs(i: Offset1 - Offset2) < CacheLineSize;
3646}
3647
3648// Set BaseReg (the base register operand), Offset (the byte offset being
3649// accessed) and the access Width of the passed instruction that reads/writes
3650// memory. Returns false if the instruction does not read/write memory or the
3651// BaseReg/Offset/Width can't be determined. Is not guaranteed to always
3652// recognise base operands and offsets in all cases.
3653// TODO: Add an IsScalable bool ref argument (like the equivalent AArch64
3654// function) and set it as appropriate.
3655bool RISCVInstrInfo::getMemOperandWithOffsetWidth(
3656 const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset,
3657 LocationSize &Width) const {
3658 if (!LdSt.mayLoadOrStore())
3659 return false;
3660
3661 // Here we assume the standard RISC-V ISA, which uses a base+offset
3662 // addressing mode. You'll need to relax these conditions to support custom
3663 // load/store instructions.
3664 if (LdSt.getNumExplicitOperands() != 3)
3665 return false;
3666 if ((!LdSt.getOperand(i: 1).isReg() && !LdSt.getOperand(i: 1).isFI()) ||
3667 !LdSt.getOperand(i: 2).isImm())
3668 return false;
3669
3670 if (!LdSt.hasOneMemOperand())
3671 return false;
3672
3673 Width = (*LdSt.memoperands_begin())->getSize();
3674 BaseReg = &LdSt.getOperand(i: 1);
3675 Offset = LdSt.getOperand(i: 2).getImm();
3676 return true;
3677}
3678
3679bool RISCVInstrInfo::areMemAccessesTriviallyDisjoint(
3680 const MachineInstr &MIa, const MachineInstr &MIb) const {
3681 assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
3682 assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
3683
3684 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() ||
3685 MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
3686 return false;
3687
3688 // Retrieve the base register, offset from the base register and width. Width
3689 // is the size of memory that is being loaded/stored (e.g. 1, 2, 4). If
3690 // base registers are identical, and the offset of a lower memory access +
3691 // the width doesn't overlap the offset of a higher memory access,
3692 // then the memory accesses are different.
3693 const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
3694 int64_t OffsetA = 0, OffsetB = 0;
3695 LocationSize WidthA = LocationSize::precise(Value: 0),
3696 WidthB = LocationSize::precise(Value: 0);
3697 if (getMemOperandWithOffsetWidth(LdSt: MIa, BaseReg&: BaseOpA, Offset&: OffsetA, Width&: WidthA) &&
3698 getMemOperandWithOffsetWidth(LdSt: MIb, BaseReg&: BaseOpB, Offset&: OffsetB, Width&: WidthB)) {
3699 if (BaseOpA->isIdenticalTo(Other: *BaseOpB)) {
3700 int LowOffset = std::min(a: OffsetA, b: OffsetB);
3701 int HighOffset = std::max(a: OffsetA, b: OffsetB);
3702 LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
3703 if (LowWidth.hasValue() &&
3704 LowOffset + (int)LowWidth.getValue() <= HighOffset)
3705 return true;
3706 }
3707 }
3708 return false;
3709}
3710
3711std::pair<unsigned, unsigned>
3712RISCVInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
3713 const unsigned Mask = RISCVII::MO_DIRECT_FLAG_MASK;
3714 return std::make_pair(x: TF & Mask, y: TF & ~Mask);
3715}
3716
3717ArrayRef<std::pair<unsigned, const char *>>
3718RISCVInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
3719 using namespace RISCVII;
3720 static const std::pair<unsigned, const char *> TargetFlags[] = {
3721 {MO_CALL, "riscv-call"},
3722 {MO_LO, "riscv-lo"},
3723 {MO_HI, "riscv-hi"},
3724 {MO_PCREL_LO, "riscv-pcrel-lo"},
3725 {MO_PCREL_HI, "riscv-pcrel-hi"},
3726 {MO_GOT_HI, "riscv-got-hi"},
3727 {MO_TPREL_LO, "riscv-tprel-lo"},
3728 {MO_TPREL_HI, "riscv-tprel-hi"},
3729 {MO_TPREL_ADD, "riscv-tprel-add"},
3730 {MO_TLS_GOT_HI, "riscv-tls-got-hi"},
3731 {MO_TLS_GD_HI, "riscv-tls-gd-hi"},
3732 {MO_TLSDESC_HI, "riscv-tlsdesc-hi"},
3733 {MO_TLSDESC_LOAD_LO, "riscv-tlsdesc-load-lo"},
3734 {MO_TLSDESC_ADD_LO, "riscv-tlsdesc-add-lo"},
3735 {MO_TLSDESC_CALL, "riscv-tlsdesc-call"},
3736 {MO_QC_ACCESS, "riscv-qc-access"},
3737 };
3738 return ArrayRef(TargetFlags);
3739}
3740bool RISCVInstrInfo::isFunctionSafeToOutlineFrom(
3741 MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
3742 const Function &F = MF.getFunction();
3743
3744 // Can F be deduplicated by the linker? If it can, don't outline from it.
3745 if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage())
3746 return false;
3747
3748 // Don't outline from functions with section markings; the program could
3749 // expect that all the code is in the named section.
3750 if (F.hasSection())
3751 return false;
3752
3753 // It's safe to outline from MF.
3754 return true;
3755}
3756
3757bool RISCVInstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB,
3758 unsigned &Flags) const {
3759 // More accurate safety checking is done in getOutliningCandidateInfo.
3760 return TargetInstrInfo::isMBBSafeToOutlineFrom(MBB, Flags);
3761}
3762
3763// Enum values indicating how an outlined call should be constructed.
3764enum MachineOutlinerConstructionID {
3765 MachineOutlinerTailCall,
3766 MachineOutlinerDefault,
3767 MachineOutlinerRegSave
3768};
3769
3770bool RISCVInstrInfo::shouldOutlineFromFunctionByDefault(
3771 MachineFunction &MF) const {
3772 return MF.getFunction().hasMinSize();
3773}
3774
3775static bool isCandidatePatchable(const MachineBasicBlock &MBB) {
3776 const MachineFunction *MF = MBB.getParent();
3777 const Function &F = MF->getFunction();
3778 return F.getFnAttribute(Kind: "fentry-call").getValueAsBool() ||
3779 F.hasFnAttribute(Kind: "patchable-function-entry");
3780}
3781
3782static bool isMIReadsReg(const MachineInstr &MI, const TargetRegisterInfo *TRI,
3783 MCRegister RegNo) {
3784 return MI.readsRegister(Reg: RegNo, TRI) ||
3785 MI.getDesc().hasImplicitUseOfPhysReg(Reg: RegNo);
3786}
3787
3788static bool isMIModifiesReg(const MachineInstr &MI,
3789 const TargetRegisterInfo *TRI, MCRegister RegNo) {
3790 return MI.modifiesRegister(Reg: RegNo, TRI) ||
3791 MI.getDesc().hasImplicitDefOfPhysReg(Reg: RegNo);
3792}
3793
3794static bool cannotInsertTailCall(const MachineBasicBlock &MBB) {
3795 if (!MBB.back().isReturn())
3796 return true;
3797 if (isCandidatePatchable(MBB))
3798 return true;
3799
3800 // If the candidate reads the pre-set register
3801 // that can be used for expanding PseudoTAIL instruction,
3802 // then we cannot insert tail call.
3803 const TargetSubtargetInfo &STI = MBB.getParent()->getSubtarget();
3804 const RISCVMachineFunctionInfo *RVFI =
3805 MBB.getParent()->getInfo<RISCVMachineFunctionInfo>();
3806 // When cf-protection-branch is active, the outliner will emit PseudoTAILX7
3807 // which always uses X7. Otherwise, PseudoTAIL is emitted and the register
3808 // is determined by Zicfilp at encode time.
3809 MCRegister TailExpandUseRegNo =
3810 RVFI->hasCFProtectionBranch()
3811 ? RISCV::X7
3812 : RISCVII::getTailExpandUseRegNo(FeatureBits: STI.getFeatureBits());
3813 for (const MachineInstr &MI : MBB) {
3814 if (isMIReadsReg(MI, TRI: STI.getRegisterInfo(), RegNo: TailExpandUseRegNo))
3815 return true;
3816 if (isMIModifiesReg(MI, TRI: STI.getRegisterInfo(), RegNo: TailExpandUseRegNo))
3817 break;
3818 }
3819 return false;
3820}
3821
3822static Register findRegisterToSaveX5To(outliner::Candidate &C,
3823 const TargetRegisterInfo &TRI) {
3824 // Candidate registers for saving X5: t1-t6
3825 static const MCPhysReg TempRegs[] = {
3826 RISCV::X6, // t1
3827 RISCV::X7, // t2
3828 RISCV::X28, // t3
3829 RISCV::X29, // t4
3830 RISCV::X30, // t5
3831 RISCV::X31 // t6
3832 };
3833
3834 const MachineFunction *MF = C.getMF();
3835 const MachineRegisterInfo &MRI = MF->getRegInfo();
3836
3837 for (MCPhysReg Reg : TempRegs) {
3838 if (MRI.isReserved(PhysReg: Reg))
3839 continue;
3840
3841 if (C.isAvailableAcrossAndOutOfSeq(Reg, TRI) &&
3842 C.isAvailableInsideSeq(Reg, TRI)) {
3843 return Reg;
3844 }
3845 }
3846
3847 return Register();
3848}
3849
3850bool RISCVInstrInfo::analyzeCandidate(outliner::Candidate &C) const {
3851 // If the expansion register for tail calls is live across the candidate
3852 // outlined call site, we cannot outline that candidate as the expansion
3853 // would clobber the register.
3854 const RISCVMachineFunctionInfo *RVFI =
3855 C.getMF()->getInfo<RISCVMachineFunctionInfo>();
3856 MCRegister TailExpandUseReg =
3857 RVFI->hasCFProtectionBranch()
3858 ? RISCV::X7
3859 : RISCVII::getTailExpandUseRegNo(FeatureBits: STI.getFeatureBits());
3860 if (C.back().isReturn() &&
3861 !C.isAvailableAcrossAndOutOfSeq(Reg: TailExpandUseReg, TRI: RegInfo)) {
3862 LLVM_DEBUG(dbgs() << "MBB:\n" << *C.getMBB());
3863 LLVM_DEBUG(dbgs() << "Cannot be outlined between: " << C.front() << "and "
3864 << C.back());
3865 LLVM_DEBUG(dbgs() << "Because the tail-call register is live across "
3866 "the proposed outlined function call\n");
3867 return true;
3868 }
3869
3870 // If last instruction is return then we can rely on
3871 // the verification already performed in the getOutliningTypeImpl.
3872 if (C.back().isReturn()) {
3873 assert(!cannotInsertTailCall(*C.getMBB()) &&
3874 "The candidate who uses return instruction must be outlined "
3875 "using tail call");
3876 return false;
3877 }
3878
3879 // Filter out candidates where the X5 register (t0) can't be used to setup
3880 // the function call.
3881 if (!C.isAvailableInsideSeq(Reg: RISCV::X5, TRI: RegInfo))
3882 return true;
3883
3884 // If X5 is available in the region, use X5 directly (MachineOutlinerDefault).
3885 if (C.isAvailableAcrossAndOutOfSeq(Reg: RISCV::X5, TRI: RegInfo))
3886 return false;
3887
3888 // Otherwise, try to save X5 into t1-t6 (MachineOutlinerRegSave).
3889 if (STI.getCLOpts().outliner_regsave && findRegisterToSaveX5To(C, TRI: RegInfo))
3890 return false;
3891
3892 return true;
3893}
3894
3895std::optional<std::unique_ptr<outliner::OutlinedFunction>>
3896RISCVInstrInfo::getOutliningCandidateInfo(
3897 const MachineModuleInfo &MMI,
3898 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
3899 unsigned MinRepeats) const {
3900
3901 // Analyze each candidate and erase the ones that are not viable.
3902 llvm::erase_if(C&: RepeatedSequenceLocs, P: [this](auto Candidate) {
3903 return analyzeCandidate(C&: Candidate);
3904 });
3905
3906 // If the sequence doesn't have enough candidates left, then we're done.
3907 if (RepeatedSequenceLocs.size() < MinRepeats)
3908 return std::nullopt;
3909
3910 // Each RepeatedSequenceLoc is identical.
3911 outliner::Candidate &Candidate = RepeatedSequenceLocs[0];
3912 unsigned InstrSizeCExt =
3913 Candidate.getMF()->getSubtarget<RISCVSubtarget>().hasStdExtZca() ? 2 : 4;
3914 unsigned CallOverhead = 0, FrameOverhead = 0;
3915
3916 // Count the number of CFI instructions in the candidate, if present.
3917 unsigned CFICount = 0;
3918 for (auto &I : Candidate) {
3919 if (I.isCFIInstruction())
3920 CFICount++;
3921 }
3922
3923 // Ensure CFI coverage matches: comparing the number of CFIs in the candidate
3924 // with the total number of CFIs in the parent function for each candidate.
3925 // Outlining only a subset of a function’s CFIs would split the unwind state
3926 // across two code regions and lead to incorrect address offsets between the
3927 // outlined body and the remaining code. To preserve correct unwind info, we
3928 // only outline when all CFIs in the function can be outlined together.
3929 for (outliner::Candidate &C : RepeatedSequenceLocs) {
3930 std::vector<MCCFIInstruction> CFIInstructions =
3931 C.getMF()->getFrameInstructions();
3932
3933 if (CFICount > 0 && CFICount != CFIInstructions.size())
3934 return std::nullopt;
3935 }
3936
3937 MachineOutlinerConstructionID MOCI = MachineOutlinerDefault;
3938 if (Candidate.back().isReturn()) {
3939 MOCI = MachineOutlinerTailCall;
3940 // tail call = auipc + jalr in the worst case without linker relaxation.
3941 // FIXME: This code suggests the JALR can be compressed - how?
3942 CallOverhead = 4 + InstrSizeCExt;
3943 // Using tail call we move ret instruction from caller to callee.
3944 FrameOverhead = 0;
3945 } else {
3946 // call t0, function = 8 bytes.
3947 CallOverhead = 8;
3948 // jr t0 = 4 bytes, 2 bytes if compressed instructions are enabled.
3949 FrameOverhead = InstrSizeCExt;
3950 }
3951
3952 // If we have CFI instructions, we can only outline if the outlined section
3953 // can be a tail call.
3954 if (MOCI != MachineOutlinerTailCall && CFICount > 0)
3955 return std::nullopt;
3956
3957 if (STI.getCLOpts().outliner_regsave && MOCI == MachineOutlinerDefault) {
3958 // Set per-candidate overhead based on X5 availability
3959 for (auto &C : RepeatedSequenceLocs) {
3960
3961 if (C.isAvailableAcrossAndOutOfSeq(Reg: RISCV::X5, TRI: RegInfo)) {
3962 // X5 is available, just need the call
3963 unsigned CandCallOverhead = 8;
3964 C.setCallInfo(CID: MachineOutlinerDefault, CO: CandCallOverhead);
3965 } else {
3966 // X5 unavailable, need save + call + restore
3967 // Save (2-4) + Call (8) + Restore (2-4)
3968 unsigned CandCallOverhead = InstrSizeCExt + 8 + InstrSizeCExt;
3969 C.setCallInfo(CID: MachineOutlinerRegSave, CO: CandCallOverhead);
3970 }
3971 }
3972 } else {
3973 for (auto &C : RepeatedSequenceLocs)
3974 C.setCallInfo(CID: MOCI, CO: CallOverhead);
3975 }
3976
3977 unsigned SequenceSize = 0;
3978 for (auto &MI : Candidate)
3979 SequenceSize += getInstSizeInBytes(MI);
3980
3981 return std::make_unique<outliner::OutlinedFunction>(
3982 args&: RepeatedSequenceLocs, args&: SequenceSize, args&: FrameOverhead, args&: MOCI);
3983}
3984
3985outliner::InstrType
3986RISCVInstrInfo::getOutliningTypeImpl(const MachineModuleInfo &MMI,
3987 MachineBasicBlock::iterator &MBBI,
3988 unsigned Flags) const {
3989 MachineInstr &MI = *MBBI;
3990 MachineBasicBlock *MBB = MI.getParent();
3991 const TargetRegisterInfo *TRI =
3992 MBB->getParent()->getSubtarget().getRegisterInfo();
3993 const auto &F = MI.getMF()->getFunction();
3994
3995 // We can only outline CFI instructions if we will tail call the outlined
3996 // function, or fix up the CFI offsets. Currently, CFI instructions are
3997 // outlined only if in a tail call.
3998 if (MI.isCFIInstruction())
3999 return outliner::InstrType::Legal;
4000
4001 if (cannotInsertTailCall(MBB: *MBB) &&
4002 (MI.isReturn() || isMIModifiesReg(MI, TRI, RegNo: RISCV::X5)))
4003 return outliner::InstrType::Illegal;
4004
4005 // Make sure the operands don't reference something unsafe.
4006 for (const auto &MO : MI.operands()) {
4007
4008 // pcrel-hi and pcrel-lo can't put in separate sections, filter that out
4009 // if any possible.
4010 if (MO.getTargetFlags() == RISCVII::MO_PCREL_LO &&
4011 (MI.getMF()->getTarget().getFunctionSections() || F.hasComdat() ||
4012 F.hasSection() || F.getSectionPrefix()))
4013 return outliner::InstrType::Illegal;
4014 }
4015
4016 if (isLPAD(MI))
4017 return outliner::InstrType::Illegal;
4018
4019 return outliner::InstrType::Legal;
4020}
4021
4022void RISCVInstrInfo::buildOutlinedFrame(
4023 MachineBasicBlock &MBB, MachineFunction &MF,
4024 const outliner::OutlinedFunction &OF) const {
4025
4026 if (OF.FrameConstructionID == MachineOutlinerTailCall)
4027 return;
4028
4029 MBB.addLiveIn(PhysReg: RISCV::X5);
4030
4031 // Add in a return instruction to the end of the outlined frame.
4032 MBB.insert(I: MBB.end(), MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: RISCV::JALR), DestReg: RISCV::X0)
4033 .addReg(RegNo: RISCV::X5)
4034 .addImm(Val: 0));
4035}
4036
4037MachineBasicBlock::iterator RISCVInstrInfo::insertOutlinedCall(
4038 Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It,
4039 MachineFunction &MF, outliner::Candidate &C) const {
4040
4041 if (C.CallConstructionID == MachineOutlinerTailCall) {
4042 const RISCVMachineFunctionInfo *RVFI =
4043 MF.getInfo<RISCVMachineFunctionInfo>();
4044 unsigned TailOpc =
4045 RVFI->hasCFProtectionBranch() ? RISCV::PseudoTAILX7 : RISCV::PseudoTAIL;
4046 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: TailOpc))
4047 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName()),
4048 /*Offset=*/0, TargetFlags: RISCVII::MO_CALL));
4049 return It;
4050 }
4051
4052 if (C.CallConstructionID == MachineOutlinerRegSave) {
4053 Register SaveReg = findRegisterToSaveX5To(C, TRI: RegInfo);
4054 assert(SaveReg && "Cannot find an available register to save/restore X5.");
4055
4056 // Save: ADDI SaveReg, X5, 0 (equivalent to MV SaveReg, X5)
4057 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: RISCV::ADDI), DestReg: SaveReg)
4058 .addReg(RegNo: RISCV::X5)
4059 .addImm(Val: 0));
4060 It++;
4061
4062 // Call: PseudoCALLReg X5
4063 It = MBB.insert(
4064 I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: RISCV::PseudoCALLReg), DestReg: RISCV::X5)
4065 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName()), Offset: 0,
4066 TargetFlags: RISCVII::MO_CALL));
4067 MachineBasicBlock::iterator CallPt = It;
4068 It++;
4069
4070 // Restore: ADDI X5, SaveReg, 0 (equivalent to MV X5, SaveReg)
4071 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: RISCV::ADDI), DestReg: RISCV::X5)
4072 .addReg(RegNo: SaveReg)
4073 .addImm(Val: 0));
4074
4075 return CallPt;
4076 }
4077
4078 // Add in a call instruction to the outlined function at the given location.
4079 It = MBB.insert(I: It,
4080 MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: RISCV::PseudoCALLReg), DestReg: RISCV::X5)
4081 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName()), Offset: 0,
4082 TargetFlags: RISCVII::MO_CALL));
4083 return It;
4084}
4085
4086void RISCVInstrInfo::buildClearRegister(Register Reg, MachineBasicBlock &MBB,
4087 MachineBasicBlock::iterator Iter,
4088 DebugLoc &DL,
4089 bool AllowSideEffects) const {
4090
4091 const MachineFunction &MF = *MBB.getParent();
4092 const RISCVRegisterInfo &TRI = *STI.getRegisterInfo();
4093
4094 if (TRI.isGeneralPurposeRegister(MF, Reg)) {
4095 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: RISCV::PseudoClearGPR), DestReg: Reg);
4096 } else if (RISCV::FPR32RegClass.contains(Reg)) {
4097 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: RISCV::PseudoClearFPR32), DestReg: Reg);
4098 } else if (RISCV::FPR64RegClass.contains(Reg)) {
4099 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: RISCV::PseudoClearFPR64), DestReg: Reg);
4100 } else if (RISCV::FPR128RegClass.contains(Reg)) {
4101 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: RISCV::PseudoClearFPR128), DestReg: Reg);
4102 } else if (RISCV::VRRegClass.contains(Reg)) {
4103 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: RISCV::PseudoClearVR), DestReg: Reg);
4104 } else {
4105 llvm::reportFatalInternalError(
4106 reason: "buildClearRegister is not implemented for " + TRI.getRegAsmName(Reg));
4107 }
4108}
4109
4110std::optional<RegImmPair> RISCVInstrInfo::isAddImmediate(const MachineInstr &MI,
4111 Register Reg) const {
4112 // TODO: Handle cases where Reg is a super- or sub-register of the
4113 // destination register.
4114 const MachineOperand &Op0 = MI.getOperand(i: 0);
4115 if (!Op0.isReg() || Reg != Op0.getReg())
4116 return std::nullopt;
4117
4118 // Don't consider ADDIW as a candidate because the caller may not be aware
4119 // of its sign extension behaviour.
4120 if (MI.getOpcode() == RISCV::ADDI && MI.getOperand(i: 1).isReg() &&
4121 MI.getOperand(i: 2).isImm())
4122 return RegImmPair{MI.getOperand(i: 1).getReg(), MI.getOperand(i: 2).getImm()};
4123
4124 return std::nullopt;
4125}
4126
4127// MIR printer helper function to annotate Operands with a comment.
4128std::string RISCVInstrInfo::createMIROperandComment(const MachineInstr &MI,
4129 const MachineOperand &Op,
4130 unsigned OpIdx) const {
4131 // Print a generic comment for this operand if there is one.
4132 std::string GenericComment =
4133 TargetInstrInfo::createMIROperandComment(MI, Op, OpIdx);
4134 if (!GenericComment.empty())
4135 return GenericComment;
4136
4137 const MCInstrDesc &Desc = MI.getDesc();
4138 if (OpIdx >= Desc.getNumOperands())
4139 return std::string();
4140
4141 std::string Comment;
4142 raw_string_ostream OS(Comment);
4143
4144 const MCOperandInfo &OpInfo = Desc.operands()[OpIdx];
4145
4146 // Print the full VType operand of vsetvli/vsetivli instructions, and the SEW
4147 // operand of vector codegen pseudos.
4148 switch (OpInfo.OperandType) {
4149 case RISCVOp::OPERAND_VTYPEI10:
4150 case RISCVOp::OPERAND_VTYPEI11: {
4151 unsigned Imm = Op.getImm();
4152 RISCVVType::printVType(VType: Imm, OS);
4153 break;
4154 }
4155 case RISCVOp::OPERAND_XSFMM_VTYPE: {
4156 unsigned Imm = Op.getImm();
4157 RISCVVType::printXSfmmVType(VType: Imm, OS);
4158 break;
4159 }
4160 case RISCVOp::OPERAND_XSFMM_TWIDEN: {
4161 unsigned Imm = Op.getImm();
4162 OS << "w" << Imm;
4163 break;
4164 }
4165 case RISCVOp::OPERAND_SEW:
4166 case RISCVOp::OPERAND_SEW_MASK: {
4167 unsigned Log2SEW = Op.getImm();
4168 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
4169 assert(RISCVVType::isValidSEW(SEW) && "Unexpected SEW");
4170 OS << "e" << SEW;
4171 break;
4172 }
4173 case RISCVOp::OPERAND_VEC_POLICY: {
4174 unsigned Policy = Op.getImm();
4175 assert(Policy <= (RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC) &&
4176 "Invalid Policy Value");
4177 OS << (Policy & RISCVVType::TAIL_AGNOSTIC ? "ta" : "tu") << ", "
4178 << (Policy & RISCVVType::MASK_AGNOSTIC ? "ma" : "mu");
4179 break;
4180 }
4181 case RISCVOp::OPERAND_AVL:
4182 if (Op.isImm() && Op.getImm() == -1)
4183 OS << "vl=VLMAX";
4184 else
4185 OS << "vl";
4186 break;
4187 case RISCVOp::OPERAND_VEC_RM:
4188 if (RISCVII::usesVXRM(TSFlags: Desc.TSFlags)) {
4189 assert(RISCVVXRndMode::isValidRoundingMode(Op.getImm()));
4190 auto VXRM = static_cast<RISCVVXRndMode::RoundingMode>(Op.getImm());
4191 OS << "vxrm=" << RISCVVXRndMode::roundingModeToString(RndMode: VXRM);
4192 } else {
4193 assert(RISCVFPRndMode::isValidRoundingMode(Op.getImm()));
4194 auto FRM = static_cast<RISCVFPRndMode::RoundingMode>(Op.getImm());
4195 OS << "frm=" << RISCVFPRndMode::roundingModeToString(RndMode: FRM);
4196 }
4197 break;
4198 }
4199
4200 return Comment;
4201}
4202
4203// clang-format off
4204#define CASE_RVV_OPCODE_UNMASK_LMUL(OP, LMUL) \
4205 RISCV::Pseudo##OP##_##LMUL
4206
4207#define CASE_RVV_OPCODE_MASK_LMUL(OP, LMUL) \
4208 RISCV::Pseudo##OP##_##LMUL##_MASK
4209
4210#define CASE_RVV_OPCODE_LMUL(OP, LMUL) \
4211 CASE_RVV_OPCODE_UNMASK_LMUL(OP, LMUL): \
4212 case CASE_RVV_OPCODE_MASK_LMUL(OP, LMUL)
4213
4214#define CASE_RVV_OPCODE_UNMASK_WIDEN(OP) \
4215 CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF8): \
4216 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF4): \
4217 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF2): \
4218 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M1): \
4219 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M2): \
4220 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M4)
4221
4222#define CASE_RVV_OPCODE_UNMASK(OP) \
4223 CASE_RVV_OPCODE_UNMASK_WIDEN(OP): \
4224 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M8)
4225
4226#define CASE_RVV_OPCODE_MASK_WIDEN(OP) \
4227 CASE_RVV_OPCODE_MASK_LMUL(OP, MF8): \
4228 case CASE_RVV_OPCODE_MASK_LMUL(OP, MF4): \
4229 case CASE_RVV_OPCODE_MASK_LMUL(OP, MF2): \
4230 case CASE_RVV_OPCODE_MASK_LMUL(OP, M1): \
4231 case CASE_RVV_OPCODE_MASK_LMUL(OP, M2): \
4232 case CASE_RVV_OPCODE_MASK_LMUL(OP, M4)
4233
4234#define CASE_RVV_OPCODE_MASK(OP) \
4235 CASE_RVV_OPCODE_MASK_WIDEN(OP): \
4236 case CASE_RVV_OPCODE_MASK_LMUL(OP, M8)
4237
4238#define CASE_RVV_OPCODE_WIDEN(OP) \
4239 CASE_RVV_OPCODE_UNMASK_WIDEN(OP): \
4240 case CASE_RVV_OPCODE_MASK_WIDEN(OP)
4241
4242#define CASE_RVV_OPCODE(OP) \
4243 CASE_RVV_OPCODE_UNMASK(OP): \
4244 case CASE_RVV_OPCODE_MASK(OP)
4245// clang-format on
4246
4247// clang-format off
4248#define CASE_VMA_OPCODE_COMMON(OP, TYPE, LMUL) \
4249 RISCV::PseudoV##OP##_##TYPE##_##LMUL
4250
4251#define CASE_VMA_OPCODE_LMULS(OP, TYPE) \
4252 CASE_VMA_OPCODE_COMMON(OP, TYPE, MF8): \
4253 case CASE_VMA_OPCODE_COMMON(OP, TYPE, MF4): \
4254 case CASE_VMA_OPCODE_COMMON(OP, TYPE, MF2): \
4255 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M1): \
4256 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M2): \
4257 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M4): \
4258 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M8)
4259
4260// VFMA instructions are SEW specific.
4261#define CASE_VFMA_OPCODE_COMMON(OP, TYPE, LMUL, SEW) \
4262 RISCV::PseudoV##OP##_##TYPE##_##LMUL##_##SEW
4263
4264#define CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE, SEW) \
4265 CASE_VFMA_OPCODE_COMMON(OP, TYPE, M1, SEW): \
4266 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M2, SEW): \
4267 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M4, SEW): \
4268 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M8, SEW)
4269
4270#define CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE, SEW) \
4271 CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF2, SEW): \
4272 case CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE, SEW)
4273
4274#define CASE_VFMA_OPCODE_LMULS_MF4(OP, TYPE, SEW) \
4275 CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF4, SEW): \
4276 case CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE, SEW)
4277
4278#define CASE_VFMA_OPCODE_VV(OP) \
4279 CASE_VFMA_OPCODE_LMULS_MF4(OP, VV, E16): \
4280 case CASE_VFMA_OPCODE_LMULS_MF4(OP##_ALT, VV, E16): \
4281 case CASE_VFMA_OPCODE_LMULS_MF2(OP, VV, E32): \
4282 case CASE_VFMA_OPCODE_LMULS_M1(OP, VV, E64)
4283
4284#define CASE_VFMA_SPLATS(OP) \
4285 CASE_VFMA_OPCODE_LMULS_MF4(OP, VFPR16, E16): \
4286 case CASE_VFMA_OPCODE_LMULS_MF4(OP##_ALT, VFPR16, E16): \
4287 case CASE_VFMA_OPCODE_LMULS_MF2(OP, VFPR32, E32): \
4288 case CASE_VFMA_OPCODE_LMULS_M1(OP, VFPR64, E64)
4289// clang-format on
4290
4291bool RISCVInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
4292 unsigned &SrcOpIdx1,
4293 unsigned &SrcOpIdx2) const {
4294 const MCInstrDesc &Desc = MI.getDesc();
4295 if (!Desc.isCommutable())
4296 return false;
4297
4298 switch (MI.getOpcode()) {
4299 case RISCV::TH_MVEQZ:
4300 case RISCV::TH_MVNEZ:
4301 // We can't commute operands if operand 2 (i.e., rs1 in
4302 // mveqz/mvnez rd,rs1,rs2) is the zero-register (as it is
4303 // not valid as the in/out-operand 1).
4304 if (MI.getOperand(i: 2).getReg() == RISCV::X0)
4305 return false;
4306 // Operands 1 and 2 are commutable, if we switch the opcode.
4307 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 1, CommutableOpIdx2: 2);
4308 case RISCV::QC_SELECTIEQ:
4309 case RISCV::QC_SELECTINE:
4310 case RISCV::QC_SELECTIIEQ:
4311 case RISCV::QC_SELECTIINE:
4312 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 1, CommutableOpIdx2: 2);
4313 case RISCV::QC_MVEQ:
4314 case RISCV::QC_MVNE:
4315 case RISCV::QC_MVLT:
4316 case RISCV::QC_MVGE:
4317 case RISCV::QC_MVLTU:
4318 case RISCV::QC_MVGEU:
4319 case RISCV::QC_MVEQI:
4320 case RISCV::QC_MVNEI:
4321 case RISCV::QC_MVLTI:
4322 case RISCV::QC_MVGEI:
4323 case RISCV::QC_MVLTUI:
4324 case RISCV::QC_MVGEUI:
4325 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 1, CommutableOpIdx2: 4);
4326 case RISCV::TH_MULA:
4327 case RISCV::TH_MULAW:
4328 case RISCV::TH_MULAH:
4329 case RISCV::TH_MULS:
4330 case RISCV::TH_MULSW:
4331 case RISCV::TH_MULSH:
4332 // Operands 2 and 3 are commutable.
4333 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 2, CommutableOpIdx2: 3);
4334 case RISCV::PseudoCCMOVGPRNoX0:
4335 case RISCV::PseudoCCMOVGPR:
4336 // Operands 1 and 2 are commutable.
4337 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 1, CommutableOpIdx2: 2);
4338 case CASE_RVV_OPCODE(VADD_VV):
4339 case CASE_RVV_OPCODE(VAND_VV):
4340 case CASE_RVV_OPCODE(VOR_VV):
4341 case CASE_RVV_OPCODE(VXOR_VV):
4342 case CASE_RVV_OPCODE_MASK(VMSEQ_VV):
4343 case CASE_RVV_OPCODE_MASK(VMSNE_VV):
4344 case CASE_RVV_OPCODE(VMIN_VV):
4345 case CASE_RVV_OPCODE(VMINU_VV):
4346 case CASE_RVV_OPCODE(VMAX_VV):
4347 case CASE_RVV_OPCODE(VMAXU_VV):
4348 case CASE_RVV_OPCODE(VMUL_VV):
4349 case CASE_RVV_OPCODE(VMULH_VV):
4350 case CASE_RVV_OPCODE(VMULHU_VV):
4351 case CASE_RVV_OPCODE_WIDEN(VWADD_VV):
4352 case CASE_RVV_OPCODE_WIDEN(VWADDU_VV):
4353 case CASE_RVV_OPCODE_WIDEN(VWMUL_VV):
4354 case CASE_RVV_OPCODE_WIDEN(VWMULU_VV):
4355 case CASE_RVV_OPCODE_WIDEN(VWMACC_VV):
4356 case CASE_RVV_OPCODE_WIDEN(VWMACCU_VV):
4357 case CASE_RVV_OPCODE(VABD_VV):
4358 case CASE_RVV_OPCODE(VABDU_VV):
4359 case CASE_RVV_OPCODE_WIDEN(VWABDA_VV):
4360 case CASE_RVV_OPCODE_WIDEN(VWABDAU_VV):
4361 case CASE_RVV_OPCODE_UNMASK(VADC_VVM):
4362 case CASE_RVV_OPCODE(VSADD_VV):
4363 case CASE_RVV_OPCODE(VSADDU_VV):
4364 case CASE_RVV_OPCODE(VAADD_VV):
4365 case CASE_RVV_OPCODE(VAADDU_VV):
4366 case CASE_RVV_OPCODE(VSMUL_VV):
4367 case CASE_RVV_OPCODE_LMUL(VDOT4A_VV, MF2):
4368 case CASE_RVV_OPCODE_LMUL(VDOT4A_VV, M1):
4369 case CASE_RVV_OPCODE_LMUL(VDOT4A_VV, M2):
4370 case CASE_RVV_OPCODE_LMUL(VDOT4A_VV, M4):
4371 case CASE_RVV_OPCODE_LMUL(VDOT4A_VV, M8):
4372 case CASE_RVV_OPCODE_LMUL(VDOT4AU_VV, MF2):
4373 case CASE_RVV_OPCODE_LMUL(VDOT4AU_VV, M1):
4374 case CASE_RVV_OPCODE_LMUL(VDOT4AU_VV, M2):
4375 case CASE_RVV_OPCODE_LMUL(VDOT4AU_VV, M4):
4376 case CASE_RVV_OPCODE_LMUL(VDOT4AU_VV, M8):
4377 // Operands 2 and 3 are commutable.
4378 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 2, CommutableOpIdx2: 3);
4379 case CASE_VFMA_SPLATS(FMADD):
4380 case CASE_VFMA_SPLATS(FMSUB):
4381 case CASE_VFMA_SPLATS(FMACC):
4382 case CASE_VFMA_SPLATS(FMSAC):
4383 case CASE_VFMA_SPLATS(FNMADD):
4384 case CASE_VFMA_SPLATS(FNMSUB):
4385 case CASE_VFMA_SPLATS(FNMACC):
4386 case CASE_VFMA_SPLATS(FNMSAC):
4387 case CASE_VFMA_OPCODE_VV(FMACC):
4388 case CASE_VFMA_OPCODE_VV(FMSAC):
4389 case CASE_VFMA_OPCODE_VV(FNMACC):
4390 case CASE_VFMA_OPCODE_VV(FNMSAC):
4391 case CASE_VMA_OPCODE_LMULS(MADD, VX):
4392 case CASE_VMA_OPCODE_LMULS(NMSUB, VX):
4393 case CASE_VMA_OPCODE_LMULS(MACC, VX):
4394 case CASE_VMA_OPCODE_LMULS(NMSAC, VX):
4395 case CASE_VMA_OPCODE_LMULS(MACC, VV):
4396 case CASE_VMA_OPCODE_LMULS(NMSAC, VV): {
4397 // If the tail policy is undisturbed we can't commute.
4398 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags));
4399 if ((MI.getOperand(i: RISCVII::getVecPolicyOpNum(Desc: MI.getDesc())).getImm() &
4400 1) == 0)
4401 return false;
4402
4403 // For these instructions we can only swap operand 1 and operand 3 by
4404 // changing the opcode.
4405 unsigned CommutableOpIdx1 = 1;
4406 unsigned CommutableOpIdx2 = 3;
4407 if (!fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1,
4408 CommutableOpIdx2))
4409 return false;
4410 return true;
4411 }
4412 case CASE_VFMA_OPCODE_VV(FMADD):
4413 case CASE_VFMA_OPCODE_VV(FMSUB):
4414 case CASE_VFMA_OPCODE_VV(FNMADD):
4415 case CASE_VFMA_OPCODE_VV(FNMSUB):
4416 case CASE_VMA_OPCODE_LMULS(MADD, VV):
4417 case CASE_VMA_OPCODE_LMULS(NMSUB, VV): {
4418 // If the tail policy is undisturbed we can't commute.
4419 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags));
4420 if ((MI.getOperand(i: RISCVII::getVecPolicyOpNum(Desc: MI.getDesc())).getImm() &
4421 1) == 0)
4422 return false;
4423
4424 // For these instructions we have more freedom. We can commute with the
4425 // other multiplicand or with the addend/subtrahend/minuend.
4426
4427 // Any fixed operand must be from source 1, 2 or 3.
4428 if (SrcOpIdx1 != CommuteAnyOperandIndex && SrcOpIdx1 > 3)
4429 return false;
4430 if (SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx2 > 3)
4431 return false;
4432
4433 // It both ops are fixed one must be the tied source.
4434 if (SrcOpIdx1 != CommuteAnyOperandIndex &&
4435 SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx1 != 1 && SrcOpIdx2 != 1)
4436 return false;
4437
4438 // Look for two different register operands assumed to be commutable
4439 // regardless of the FMA opcode. The FMA opcode is adjusted later if
4440 // needed.
4441 if (SrcOpIdx1 == CommuteAnyOperandIndex ||
4442 SrcOpIdx2 == CommuteAnyOperandIndex) {
4443 // At least one of operands to be commuted is not specified and
4444 // this method is free to choose appropriate commutable operands.
4445 unsigned CommutableOpIdx1 = SrcOpIdx1;
4446 if (SrcOpIdx1 == SrcOpIdx2) {
4447 // Both of operands are not fixed. Set one of commutable
4448 // operands to the tied source.
4449 CommutableOpIdx1 = 1;
4450 } else if (SrcOpIdx1 == CommuteAnyOperandIndex) {
4451 // Only one of the operands is not fixed.
4452 CommutableOpIdx1 = SrcOpIdx2;
4453 }
4454
4455 // CommutableOpIdx1 is well defined now. Let's choose another commutable
4456 // operand and assign its index to CommutableOpIdx2.
4457 unsigned CommutableOpIdx2;
4458 if (CommutableOpIdx1 != 1) {
4459 // If we haven't already used the tied source, we must use it now.
4460 CommutableOpIdx2 = 1;
4461 } else {
4462 Register Op1Reg = MI.getOperand(i: CommutableOpIdx1).getReg();
4463
4464 // The commuted operands should have different registers.
4465 // Otherwise, the commute transformation does not change anything and
4466 // is useless. We use this as a hint to make our decision.
4467 if (Op1Reg != MI.getOperand(i: 2).getReg())
4468 CommutableOpIdx2 = 2;
4469 else
4470 CommutableOpIdx2 = 3;
4471 }
4472
4473 // Assign the found pair of commutable indices to SrcOpIdx1 and
4474 // SrcOpIdx2 to return those values.
4475 if (!fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1,
4476 CommutableOpIdx2))
4477 return false;
4478 }
4479
4480 return true;
4481 }
4482 }
4483
4484 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
4485}
4486
4487// clang-format off
4488#define CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL) \
4489 case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL: \
4490 Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL; \
4491 break;
4492
4493#define CASE_VMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE) \
4494 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF8) \
4495 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4) \
4496 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2) \
4497 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1) \
4498 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2) \
4499 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4) \
4500 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8)
4501
4502// VFMA depends on SEW.
4503#define CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL, SEW) \
4504 case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL##_##SEW: \
4505 Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL##_##SEW; \
4506 break;
4507
4508#define CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE, SEW) \
4509 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1, SEW) \
4510 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2, SEW) \
4511 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4, SEW) \
4512 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8, SEW)
4513
4514#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE, SEW) \
4515 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2, SEW) \
4516 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE, SEW)
4517
4518#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE, SEW) \
4519 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4, SEW) \
4520 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE, SEW)
4521
4522#define CASE_VFMA_CHANGE_OPCODE_VV(OLDOP, NEWOP) \
4523 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VV, E16) \
4524 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP##_ALT, NEWOP##_ALT, VV, E16) \
4525 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VV, E32) \
4526 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VV, E64)
4527
4528#define CASE_VFMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP) \
4529 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VFPR16, E16) \
4530 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP##_ALT, NEWOP##_ALT, VFPR16, E16) \
4531 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VFPR32, E32) \
4532 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VFPR64, E64)
4533// clang-format on
4534
4535MachineInstr *RISCVInstrInfo::commuteInstructionImpl(MachineInstr &MI,
4536 bool NewMI,
4537 unsigned OpIdx1,
4538 unsigned OpIdx2) const {
4539 auto cloneIfNew = [NewMI](MachineInstr &MI) -> MachineInstr & {
4540 if (NewMI)
4541 return *MI.getParent()->getParent()->CloneMachineInstr(Orig: &MI);
4542 return MI;
4543 };
4544
4545 switch (MI.getOpcode()) {
4546 case RISCV::TH_MVEQZ:
4547 case RISCV::TH_MVNEZ: {
4548 auto &WorkingMI = cloneIfNew(MI);
4549 WorkingMI.setDesc(get(Opcode: MI.getOpcode() == RISCV::TH_MVEQZ ? RISCV::TH_MVNEZ
4550 : RISCV::TH_MVEQZ));
4551 return TargetInstrInfo::commuteInstructionImpl(MI&: WorkingMI, NewMI: false, OpIdx1,
4552 OpIdx2);
4553 }
4554 case RISCV::QC_SELECTIEQ:
4555 case RISCV::QC_SELECTINE:
4556 case RISCV::QC_SELECTIIEQ:
4557 case RISCV::QC_SELECTIINE:
4558 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
4559 case RISCV::QC_MVEQ:
4560 case RISCV::QC_MVNE:
4561 case RISCV::QC_MVLT:
4562 case RISCV::QC_MVGE:
4563 case RISCV::QC_MVLTU:
4564 case RISCV::QC_MVGEU:
4565 case RISCV::QC_MVEQI:
4566 case RISCV::QC_MVNEI:
4567 case RISCV::QC_MVLTI:
4568 case RISCV::QC_MVGEI:
4569 case RISCV::QC_MVLTUI:
4570 case RISCV::QC_MVGEUI: {
4571 auto &WorkingMI = cloneIfNew(MI);
4572 WorkingMI.setDesc(get(Opcode: getInverseXqcicmOpcode(Opcode: MI.getOpcode())));
4573 return TargetInstrInfo::commuteInstructionImpl(MI&: WorkingMI, NewMI: false, OpIdx1,
4574 OpIdx2);
4575 }
4576 case RISCV::PseudoCCMOVGPRNoX0:
4577 case RISCV::PseudoCCMOVGPR: {
4578 // CCMOV can be commuted by inverting the condition.
4579 unsigned BCC = MI.getOperand(i: MI.getNumExplicitOperands() - 3).getImm();
4580 BCC = RISCVCC::getInverseBranchOpcode(BCC);
4581 auto &WorkingMI = cloneIfNew(MI);
4582 WorkingMI.getOperand(i: MI.getNumExplicitOperands() - 3).setImm(BCC);
4583 return TargetInstrInfo::commuteInstructionImpl(MI&: WorkingMI, /*NewMI*/ false,
4584 OpIdx1, OpIdx2);
4585 }
4586 case CASE_VFMA_SPLATS(FMACC):
4587 case CASE_VFMA_SPLATS(FMADD):
4588 case CASE_VFMA_SPLATS(FMSAC):
4589 case CASE_VFMA_SPLATS(FMSUB):
4590 case CASE_VFMA_SPLATS(FNMACC):
4591 case CASE_VFMA_SPLATS(FNMADD):
4592 case CASE_VFMA_SPLATS(FNMSAC):
4593 case CASE_VFMA_SPLATS(FNMSUB):
4594 case CASE_VFMA_OPCODE_VV(FMACC):
4595 case CASE_VFMA_OPCODE_VV(FMSAC):
4596 case CASE_VFMA_OPCODE_VV(FNMACC):
4597 case CASE_VFMA_OPCODE_VV(FNMSAC):
4598 case CASE_VMA_OPCODE_LMULS(MADD, VX):
4599 case CASE_VMA_OPCODE_LMULS(NMSUB, VX):
4600 case CASE_VMA_OPCODE_LMULS(MACC, VX):
4601 case CASE_VMA_OPCODE_LMULS(NMSAC, VX):
4602 case CASE_VMA_OPCODE_LMULS(MACC, VV):
4603 case CASE_VMA_OPCODE_LMULS(NMSAC, VV): {
4604 // It only make sense to toggle these between clobbering the
4605 // addend/subtrahend/minuend one of the multiplicands.
4606 assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");
4607 assert((OpIdx1 == 3 || OpIdx2 == 3) && "Unexpected opcode index");
4608 unsigned Opc;
4609 switch (MI.getOpcode()) {
4610 default:
4611 llvm_unreachable("Unexpected opcode");
4612 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMACC, FMADD)
4613 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMADD, FMACC)
4614 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMSAC, FMSUB)
4615 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMSUB, FMSAC)
4616 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMACC, FNMADD)
4617 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMADD, FNMACC)
4618 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMSAC, FNMSUB)
4619 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMSUB, FNMSAC)
4620 CASE_VFMA_CHANGE_OPCODE_VV(FMACC, FMADD)
4621 CASE_VFMA_CHANGE_OPCODE_VV(FMSAC, FMSUB)
4622 CASE_VFMA_CHANGE_OPCODE_VV(FNMACC, FNMADD)
4623 CASE_VFMA_CHANGE_OPCODE_VV(FNMSAC, FNMSUB)
4624 CASE_VMA_CHANGE_OPCODE_LMULS(MACC, MADD, VX)
4625 CASE_VMA_CHANGE_OPCODE_LMULS(MADD, MACC, VX)
4626 CASE_VMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VX)
4627 CASE_VMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VX)
4628 CASE_VMA_CHANGE_OPCODE_LMULS(MACC, MADD, VV)
4629 CASE_VMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VV)
4630 }
4631
4632 auto &WorkingMI = cloneIfNew(MI);
4633 WorkingMI.setDesc(get(Opcode: Opc));
4634 return TargetInstrInfo::commuteInstructionImpl(MI&: WorkingMI, /*NewMI=*/false,
4635 OpIdx1, OpIdx2);
4636 }
4637 case CASE_VFMA_OPCODE_VV(FMADD):
4638 case CASE_VFMA_OPCODE_VV(FMSUB):
4639 case CASE_VFMA_OPCODE_VV(FNMADD):
4640 case CASE_VFMA_OPCODE_VV(FNMSUB):
4641 case CASE_VMA_OPCODE_LMULS(MADD, VV):
4642 case CASE_VMA_OPCODE_LMULS(NMSUB, VV): {
4643 assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");
4644 // If one of the operands, is the addend we need to change opcode.
4645 // Otherwise we're just swapping 2 of the multiplicands.
4646 if (OpIdx1 == 3 || OpIdx2 == 3) {
4647 unsigned Opc;
4648 switch (MI.getOpcode()) {
4649 default:
4650 llvm_unreachable("Unexpected opcode");
4651 CASE_VFMA_CHANGE_OPCODE_VV(FMADD, FMACC)
4652 CASE_VFMA_CHANGE_OPCODE_VV(FMSUB, FMSAC)
4653 CASE_VFMA_CHANGE_OPCODE_VV(FNMADD, FNMACC)
4654 CASE_VFMA_CHANGE_OPCODE_VV(FNMSUB, FNMSAC)
4655 CASE_VMA_CHANGE_OPCODE_LMULS(MADD, MACC, VV)
4656 CASE_VMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VV)
4657 }
4658
4659 auto &WorkingMI = cloneIfNew(MI);
4660 WorkingMI.setDesc(get(Opcode: Opc));
4661 return TargetInstrInfo::commuteInstructionImpl(MI&: WorkingMI, /*NewMI=*/false,
4662 OpIdx1, OpIdx2);
4663 }
4664 // Let the default code handle it.
4665 break;
4666 }
4667 }
4668
4669 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
4670}
4671
4672#undef CASE_VMA_CHANGE_OPCODE_COMMON
4673#undef CASE_VMA_CHANGE_OPCODE_LMULS
4674#undef CASE_VFMA_CHANGE_OPCODE_COMMON
4675#undef CASE_VFMA_CHANGE_OPCODE_LMULS_M1
4676#undef CASE_VFMA_CHANGE_OPCODE_LMULS_MF2
4677#undef CASE_VFMA_CHANGE_OPCODE_LMULS_MF4
4678#undef CASE_VFMA_CHANGE_OPCODE_VV
4679#undef CASE_VFMA_CHANGE_OPCODE_SPLATS
4680
4681#undef CASE_RVV_OPCODE_UNMASK_LMUL
4682#undef CASE_RVV_OPCODE_MASK_LMUL
4683#undef CASE_RVV_OPCODE_LMUL
4684#undef CASE_RVV_OPCODE_UNMASK_WIDEN
4685#undef CASE_RVV_OPCODE_UNMASK
4686#undef CASE_RVV_OPCODE_MASK_WIDEN
4687#undef CASE_RVV_OPCODE_MASK
4688#undef CASE_RVV_OPCODE_WIDEN
4689#undef CASE_RVV_OPCODE
4690
4691#undef CASE_VMA_OPCODE_COMMON
4692#undef CASE_VMA_OPCODE_LMULS
4693#undef CASE_VFMA_OPCODE_COMMON
4694#undef CASE_VFMA_OPCODE_LMULS_M1
4695#undef CASE_VFMA_OPCODE_LMULS_MF2
4696#undef CASE_VFMA_OPCODE_LMULS_MF4
4697#undef CASE_VFMA_OPCODE_VV
4698#undef CASE_VFMA_SPLATS
4699
4700bool RISCVInstrInfo::simplifyInstruction(MachineInstr &MI) const {
4701 switch (MI.getOpcode()) {
4702 default:
4703 break;
4704 case RISCV::ADD:
4705 case RISCV::OR:
4706 case RISCV::XOR:
4707 // Normalize (so we hit the next if clause).
4708 // add/[x]or rd, zero, rs => add/[x]or rd, rs, zero
4709 if (MI.getOperand(i: 1).getReg() == RISCV::X0)
4710 commuteInstruction(MI);
4711 // add/[x]or rd, rs, zero => addi rd, rs, 0
4712 if (MI.getOperand(i: 2).getReg() == RISCV::X0) {
4713 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4714 MI.setDesc(get(Opcode: RISCV::ADDI));
4715 return true;
4716 }
4717 // xor rd, rs, rs => addi rd, zero, 0
4718 if (MI.getOpcode() == RISCV::XOR &&
4719 MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg()) {
4720 MI.getOperand(i: 1).setReg(RISCV::X0);
4721 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4722 MI.setDesc(get(Opcode: RISCV::ADDI));
4723 return true;
4724 }
4725 break;
4726 case RISCV::ORI:
4727 case RISCV::XORI:
4728 // [x]ori rd, zero, N => addi rd, zero, N
4729 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4730 MI.setDesc(get(Opcode: RISCV::ADDI));
4731 return true;
4732 }
4733 break;
4734 case RISCV::SUB:
4735 // sub rd, rs, zero => addi rd, rs, 0
4736 if (MI.getOperand(i: 2).getReg() == RISCV::X0) {
4737 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4738 MI.setDesc(get(Opcode: RISCV::ADDI));
4739 return true;
4740 }
4741 break;
4742 case RISCV::SUBW:
4743 // subw rd, rs, zero => addiw rd, rs, 0
4744 if (MI.getOperand(i: 2).getReg() == RISCV::X0) {
4745 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4746 MI.setDesc(get(Opcode: RISCV::ADDIW));
4747 return true;
4748 }
4749 break;
4750 case RISCV::ADDW:
4751 // Normalize (so we hit the next if clause).
4752 // addw rd, zero, rs => addw rd, rs, zero
4753 if (MI.getOperand(i: 1).getReg() == RISCV::X0)
4754 commuteInstruction(MI);
4755 // addw rd, rs, zero => addiw rd, rs, 0
4756 if (MI.getOperand(i: 2).getReg() == RISCV::X0) {
4757 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4758 MI.setDesc(get(Opcode: RISCV::ADDIW));
4759 return true;
4760 }
4761 break;
4762 case RISCV::SH1ADD:
4763 case RISCV::SH1ADD_UW:
4764 case RISCV::SH2ADD:
4765 case RISCV::SH2ADD_UW:
4766 case RISCV::SH3ADD:
4767 case RISCV::SH3ADD_UW:
4768 // shNadd[.uw] rd, zero, rs => addi rd, rs, 0
4769 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4770 MI.removeOperand(OpNo: 1);
4771 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
4772 MI.setDesc(get(Opcode: RISCV::ADDI));
4773 return true;
4774 }
4775 // shNadd[.uw] rd, rs, zero => slli[.uw] rd, rs, N
4776 if (MI.getOperand(i: 2).getReg() == RISCV::X0) {
4777 MI.removeOperand(OpNo: 2);
4778 unsigned Opc = MI.getOpcode();
4779 if (Opc == RISCV::SH1ADD_UW || Opc == RISCV::SH2ADD_UW ||
4780 Opc == RISCV::SH3ADD_UW) {
4781 MI.addOperand(Op: MachineOperand::CreateImm(Val: getSHXADDUWShiftAmount(Opc)));
4782 MI.setDesc(get(Opcode: RISCV::SLLI_UW));
4783 return true;
4784 }
4785 MI.addOperand(Op: MachineOperand::CreateImm(Val: getSHXADDShiftAmount(Opc)));
4786 MI.setDesc(get(Opcode: RISCV::SLLI));
4787 return true;
4788 }
4789 break;
4790 case RISCV::AND:
4791 case RISCV::MUL:
4792 case RISCV::MULH:
4793 case RISCV::MULHSU:
4794 case RISCV::MULHU:
4795 case RISCV::MULW:
4796 // and rd, zero, rs => addi rd, zero, 0
4797 // mul* rd, zero, rs => addi rd, zero, 0
4798 // and rd, rs, zero => addi rd, zero, 0
4799 // mul* rd, rs, zero => addi rd, zero, 0
4800 if (MI.getOperand(i: 1).getReg() == RISCV::X0 ||
4801 MI.getOperand(i: 2).getReg() == RISCV::X0) {
4802 MI.getOperand(i: 1).setReg(RISCV::X0);
4803 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4804 MI.setDesc(get(Opcode: RISCV::ADDI));
4805 return true;
4806 }
4807 break;
4808 case RISCV::ANDI:
4809 // andi rd, zero, C => addi rd, zero, 0
4810 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4811 MI.getOperand(i: 2).setImm(0);
4812 MI.setDesc(get(Opcode: RISCV::ADDI));
4813 return true;
4814 }
4815 break;
4816 case RISCV::SLL:
4817 case RISCV::SRL:
4818 case RISCV::SRA:
4819 // shift rd, zero, rs => addi rd, zero, 0
4820 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4821 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4822 MI.setDesc(get(Opcode: RISCV::ADDI));
4823 return true;
4824 }
4825 // shift rd, rs, zero => addi rd, rs, 0
4826 if (MI.getOperand(i: 2).getReg() == RISCV::X0) {
4827 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4828 MI.setDesc(get(Opcode: RISCV::ADDI));
4829 return true;
4830 }
4831 break;
4832 case RISCV::SLLW:
4833 case RISCV::SRLW:
4834 case RISCV::SRAW:
4835 // shiftw rd, zero, rs => addi rd, zero, 0
4836 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4837 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4838 MI.setDesc(get(Opcode: RISCV::ADDI));
4839 return true;
4840 }
4841 break;
4842 case RISCV::SLLI:
4843 case RISCV::SRLI:
4844 case RISCV::SRAI:
4845 case RISCV::SLLIW:
4846 case RISCV::SRLIW:
4847 case RISCV::SRAIW:
4848 case RISCV::SLLI_UW:
4849 // shiftimm rd, zero, N => addi rd, zero, 0
4850 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4851 MI.getOperand(i: 2).setImm(0);
4852 MI.setDesc(get(Opcode: RISCV::ADDI));
4853 return true;
4854 }
4855 break;
4856 case RISCV::SLTU:
4857 case RISCV::ADD_UW:
4858 // sltu rd, zero, zero => addi rd, zero, 0
4859 // add.uw rd, zero, zero => addi rd, zero, 0
4860 if (MI.getOperand(i: 1).getReg() == RISCV::X0 &&
4861 MI.getOperand(i: 2).getReg() == RISCV::X0) {
4862 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4863 MI.setDesc(get(Opcode: RISCV::ADDI));
4864 return true;
4865 }
4866 // add.uw rd, zero, rs => addi rd, rs, 0
4867 if (MI.getOpcode() == RISCV::ADD_UW &&
4868 MI.getOperand(i: 1).getReg() == RISCV::X0) {
4869 MI.removeOperand(OpNo: 1);
4870 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
4871 MI.setDesc(get(Opcode: RISCV::ADDI));
4872 }
4873 break;
4874 case RISCV::SLTIU:
4875 // sltiu rd, zero, NZC => addi rd, zero, 1
4876 // sltiu rd, zero, 0 => addi rd, zero, 0
4877 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4878 MI.getOperand(i: 2).setImm(MI.getOperand(i: 2).getImm() != 0);
4879 MI.setDesc(get(Opcode: RISCV::ADDI));
4880 return true;
4881 }
4882 break;
4883 case RISCV::SEXT_H:
4884 case RISCV::SEXT_B:
4885 case RISCV::ZEXT_H_RV32:
4886 case RISCV::ZEXT_H_RV64:
4887 // sext.[hb] rd, zero => addi rd, zero, 0
4888 // zext.h rd, zero => addi rd, zero, 0
4889 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4890 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
4891 MI.setDesc(get(Opcode: RISCV::ADDI));
4892 return true;
4893 }
4894 break;
4895 case RISCV::MIN:
4896 case RISCV::MINU:
4897 case RISCV::MAX:
4898 case RISCV::MAXU:
4899 // min|max rd, rs, rs => addi rd, rs, 0
4900 if (MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg()) {
4901 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4902 MI.setDesc(get(Opcode: RISCV::ADDI));
4903 return true;
4904 }
4905 break;
4906 case RISCV::BEQ:
4907 case RISCV::BNE:
4908 // b{eq,ne} zero, rs, imm => b{eq,ne} rs, zero, imm
4909 if (MI.getOperand(i: 0).getReg() == RISCV::X0) {
4910 MachineOperand MO0 = MI.getOperand(i: 0);
4911 MI.removeOperand(OpNo: 0);
4912 MI.insert(InsertBefore: MI.operands_begin() + 1, Ops: {MO0});
4913 }
4914 break;
4915 case RISCV::BLTU:
4916 // bltu zero, rs, imm => bne rs, zero, imm
4917 if (MI.getOperand(i: 0).getReg() == RISCV::X0) {
4918 MachineOperand MO0 = MI.getOperand(i: 0);
4919 MI.removeOperand(OpNo: 0);
4920 MI.insert(InsertBefore: MI.operands_begin() + 1, Ops: {MO0});
4921 MI.setDesc(get(Opcode: RISCV::BNE));
4922 }
4923 break;
4924 case RISCV::BGEU:
4925 // bgeu zero, rs, imm => beq rs, zero, imm
4926 if (MI.getOperand(i: 0).getReg() == RISCV::X0) {
4927 MachineOperand MO0 = MI.getOperand(i: 0);
4928 MI.removeOperand(OpNo: 0);
4929 MI.insert(InsertBefore: MI.operands_begin() + 1, Ops: {MO0});
4930 MI.setDesc(get(Opcode: RISCV::BEQ));
4931 }
4932 break;
4933 }
4934 return false;
4935}
4936
4937// clang-format off
4938#define CASE_WIDEOP_OPCODE_COMMON(OP, LMUL) \
4939 RISCV::PseudoV##OP##_##LMUL##_TIED
4940
4941#define CASE_WIDEOP_OPCODE_LMULS(OP) \
4942 CASE_WIDEOP_OPCODE_COMMON(OP, MF8): \
4943 case CASE_WIDEOP_OPCODE_COMMON(OP, MF4): \
4944 case CASE_WIDEOP_OPCODE_COMMON(OP, MF2): \
4945 case CASE_WIDEOP_OPCODE_COMMON(OP, M1): \
4946 case CASE_WIDEOP_OPCODE_COMMON(OP, M2): \
4947 case CASE_WIDEOP_OPCODE_COMMON(OP, M4)
4948
4949#define CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL) \
4950 case RISCV::PseudoV##OP##_##LMUL##_TIED: \
4951 NewOpc = RISCV::PseudoV##OP##_##LMUL; \
4952 break;
4953
4954#define CASE_WIDEOP_CHANGE_OPCODE_LMULS(OP) \
4955 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF8) \
4956 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4) \
4957 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2) \
4958 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1) \
4959 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2) \
4960 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4)
4961
4962// FP Widening Ops may by SEW aware. Create SEW aware cases for these cases.
4963#define CASE_FP_WIDEOP_OPCODE_COMMON(OP, LMUL, SEW) \
4964 RISCV::PseudoV##OP##_##LMUL##_##SEW##_TIED
4965
4966#define CASE_FP_WIDEOP_OPCODE_LMULS(OP) \
4967 CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF4, E16): \
4968 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E16): \
4969 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E32): \
4970 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E16): \
4971 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E32): \
4972 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E16): \
4973 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E32): \
4974 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E16): \
4975 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E32) \
4976
4977#define CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL, SEW) \
4978 case RISCV::PseudoV##OP##_##LMUL##_##SEW##_TIED: \
4979 NewOpc = RISCV::PseudoV##OP##_##LMUL##_##SEW; \
4980 break;
4981
4982#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(OP) \
4983 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4, E16) \
4984 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E16) \
4985 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E32) \
4986 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E16) \
4987 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E32) \
4988 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E16) \
4989 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E32) \
4990 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E16) \
4991 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E32) \
4992
4993#define CASE_FP_WIDEOP_OPCODE_LMULS_ALT(OP) \
4994 CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF4, E16): \
4995 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E16): \
4996 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E16): \
4997 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E16): \
4998 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E16)
4999
5000#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_ALT(OP) \
5001 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4, E16) \
5002 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E16) \
5003 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E16) \
5004 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E16) \
5005 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E16)
5006// clang-format on
5007
5008MachineInstr *RISCVInstrInfo::convertToThreeAddress(MachineInstr &MI,
5009 LiveIntervals *LIS) const {
5010 MachineInstrBuilder MIB;
5011 switch (MI.getOpcode()) {
5012 default:
5013 return nullptr;
5014 case CASE_FP_WIDEOP_OPCODE_LMULS_ALT(FWADD_ALT_WV):
5015 case CASE_FP_WIDEOP_OPCODE_LMULS_ALT(FWSUB_ALT_WV):
5016 case CASE_FP_WIDEOP_OPCODE_LMULS(FWADD_WV):
5017 case CASE_FP_WIDEOP_OPCODE_LMULS(FWSUB_WV): {
5018 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags) &&
5019 MI.getNumExplicitOperands() == 7 &&
5020 "Expect 7 explicit operands rd, rs2, rs1, rm, vl, sew, policy");
5021 // If the tail policy is undisturbed we can't convert.
5022 if ((MI.getOperand(i: RISCVII::getVecPolicyOpNum(Desc: MI.getDesc())).getImm() &
5023 1) == 0)
5024 return nullptr;
5025 // clang-format off
5026 unsigned NewOpc;
5027 switch (MI.getOpcode()) {
5028 default:
5029 llvm_unreachable("Unexpected opcode");
5030 CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(FWADD_WV)
5031 CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(FWSUB_WV)
5032 CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_ALT(FWADD_ALT_WV)
5033 CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_ALT(FWSUB_ALT_WV)
5034 }
5035 // clang-format on
5036
5037 MachineBasicBlock &MBB = *MI.getParent();
5038 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
5039 .add(MO: MI.getOperand(i: 0))
5040 .addReg(RegNo: MI.getOperand(i: 0).getReg(), Flags: RegState::Undef)
5041 .add(MO: MI.getOperand(i: 1))
5042 .add(MO: MI.getOperand(i: 2))
5043 .add(MO: MI.getOperand(i: 3))
5044 .add(MO: MI.getOperand(i: 4))
5045 .add(MO: MI.getOperand(i: 5))
5046 .add(MO: MI.getOperand(i: 6));
5047 break;
5048 }
5049 case CASE_WIDEOP_OPCODE_LMULS(WADD_WV):
5050 case CASE_WIDEOP_OPCODE_LMULS(WADDU_WV):
5051 case CASE_WIDEOP_OPCODE_LMULS(WSUB_WV):
5052 case CASE_WIDEOP_OPCODE_LMULS(WSUBU_WV): {
5053 // If the tail policy is undisturbed we can't convert.
5054 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags) &&
5055 MI.getNumExplicitOperands() == 6);
5056 if ((MI.getOperand(i: RISCVII::getVecPolicyOpNum(Desc: MI.getDesc())).getImm() &
5057 1) == 0)
5058 return nullptr;
5059
5060 // clang-format off
5061 unsigned NewOpc;
5062 switch (MI.getOpcode()) {
5063 default:
5064 llvm_unreachable("Unexpected opcode");
5065 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WADD_WV)
5066 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WADDU_WV)
5067 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WSUB_WV)
5068 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WSUBU_WV)
5069 }
5070 // clang-format on
5071
5072 MachineBasicBlock &MBB = *MI.getParent();
5073 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
5074 .add(MO: MI.getOperand(i: 0))
5075 .addReg(RegNo: MI.getOperand(i: 0).getReg(), Flags: RegState::Undef)
5076 .add(MO: MI.getOperand(i: 1))
5077 .add(MO: MI.getOperand(i: 2))
5078 .add(MO: MI.getOperand(i: 3))
5079 .add(MO: MI.getOperand(i: 4))
5080 .add(MO: MI.getOperand(i: 5));
5081 break;
5082 }
5083 }
5084 MIB.copyImplicitOps(OtherMI: MI);
5085
5086 if (LIS) {
5087 SlotIndex Idx = LIS->ReplaceMachineInstrInMaps(MI, NewMI&: *MIB);
5088
5089 if (MI.getOperand(i: 0).isEarlyClobber()) {
5090 // Use operand 1 was tied to early-clobber def operand 0, so its live
5091 // interval could have ended at an early-clobber slot. Now they are not
5092 // tied we need to update it to the normal register slot.
5093 LiveInterval &LI = LIS->getInterval(Reg: MI.getOperand(i: 1).getReg());
5094 LiveRange::Segment *S = LI.getSegmentContaining(Idx);
5095 if (S->end == Idx.getRegSlot(EC: true))
5096 S->end = Idx.getRegSlot();
5097 }
5098 }
5099
5100 return MIB;
5101}
5102
5103#undef CASE_WIDEOP_OPCODE_COMMON
5104#undef CASE_WIDEOP_OPCODE_LMULS
5105#undef CASE_WIDEOP_CHANGE_OPCODE_COMMON
5106#undef CASE_WIDEOP_CHANGE_OPCODE_LMULS
5107#undef CASE_FP_WIDEOP_OPCODE_COMMON
5108#undef CASE_FP_WIDEOP_OPCODE_LMULS
5109#undef CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON
5110#undef CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS
5111
5112void RISCVInstrInfo::mulImm(MachineFunction &MF, MachineBasicBlock &MBB,
5113 MachineBasicBlock::iterator II, const DebugLoc &DL,
5114 Register DestReg, uint32_t Amount,
5115 MachineInstr::MIFlag Flag) const {
5116 MachineRegisterInfo &MRI = MF.getRegInfo();
5117 if (llvm::has_single_bit(Value: Amount)) {
5118 uint32_t ShiftAmount = Log2_32(Value: Amount);
5119 if (ShiftAmount == 0)
5120 return;
5121 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::SLLI), DestReg)
5122 .addReg(RegNo: DestReg, Flags: RegState::Kill)
5123 .addImm(Val: ShiftAmount)
5124 .setMIFlag(Flag);
5125 } else if (int ShXAmount, ShiftAmount;
5126 STI.hasShlAdd(ShAmt: 3) &&
5127 (ShXAmount = isShifted359(Value: Amount, Shift&: ShiftAmount)) != 0) {
5128 // We can use Zba SHXADD+SLLI instructions for multiply in some cases.
5129 unsigned Opc;
5130 switch (ShXAmount) {
5131 case 1:
5132 Opc = RISCV::SH1ADD;
5133 break;
5134 case 2:
5135 Opc = RISCV::SH2ADD;
5136 break;
5137 case 3:
5138 Opc = RISCV::SH3ADD;
5139 break;
5140 default:
5141 llvm_unreachable("unexpected result of isShifted359");
5142 }
5143 if (ShiftAmount)
5144 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::SLLI), DestReg)
5145 .addReg(RegNo: DestReg, Flags: RegState::Kill)
5146 .addImm(Val: ShiftAmount)
5147 .setMIFlag(Flag);
5148 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: Opc), DestReg)
5149 .addReg(RegNo: DestReg, Flags: RegState::Kill)
5150 .addReg(RegNo: DestReg)
5151 .setMIFlag(Flag);
5152 } else if (llvm::has_single_bit(Value: Amount - 1)) {
5153 Register ScaledRegister = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
5154 uint32_t ShiftAmount = Log2_32(Value: Amount - 1);
5155 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::SLLI), DestReg: ScaledRegister)
5156 .addReg(RegNo: DestReg)
5157 .addImm(Val: ShiftAmount)
5158 .setMIFlag(Flag);
5159 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::ADD), DestReg)
5160 .addReg(RegNo: ScaledRegister, Flags: RegState::Kill)
5161 .addReg(RegNo: DestReg, Flags: RegState::Kill)
5162 .setMIFlag(Flag);
5163 } else if (llvm::has_single_bit(Value: Amount + 1)) {
5164 Register ScaledRegister = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
5165 uint32_t ShiftAmount = Log2_32(Value: Amount + 1);
5166 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::SLLI), DestReg: ScaledRegister)
5167 .addReg(RegNo: DestReg)
5168 .addImm(Val: ShiftAmount)
5169 .setMIFlag(Flag);
5170 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::SUB), DestReg)
5171 .addReg(RegNo: ScaledRegister, Flags: RegState::Kill)
5172 .addReg(RegNo: DestReg, Flags: RegState::Kill)
5173 .setMIFlag(Flag);
5174 } else if (STI.hasStdExtZmmul()) {
5175 Register N = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
5176 movImm(MBB, MBBI: II, DL, DstReg: N, Val: Amount, Flag);
5177 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::MUL), DestReg)
5178 .addReg(RegNo: DestReg, Flags: RegState::Kill)
5179 .addReg(RegNo: N, Flags: RegState::Kill)
5180 .setMIFlag(Flag);
5181 } else {
5182 Register Acc;
5183 uint32_t PrevShiftAmount = 0;
5184 for (uint32_t ShiftAmount = 0; Amount >> ShiftAmount; ShiftAmount++) {
5185 if (Amount & (1U << ShiftAmount)) {
5186 if (ShiftAmount)
5187 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::SLLI), DestReg)
5188 .addReg(RegNo: DestReg, Flags: RegState::Kill)
5189 .addImm(Val: ShiftAmount - PrevShiftAmount)
5190 .setMIFlag(Flag);
5191 if (Amount >> (ShiftAmount + 1)) {
5192 // If we don't have an accmulator yet, create it and copy DestReg.
5193 if (!Acc) {
5194 Acc = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
5195 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: TargetOpcode::COPY), DestReg: Acc)
5196 .addReg(RegNo: DestReg)
5197 .setMIFlag(Flag);
5198 } else {
5199 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::ADD), DestReg: Acc)
5200 .addReg(RegNo: Acc, Flags: RegState::Kill)
5201 .addReg(RegNo: DestReg)
5202 .setMIFlag(Flag);
5203 }
5204 }
5205 PrevShiftAmount = ShiftAmount;
5206 }
5207 }
5208 assert(Acc && "Expected valid accumulator");
5209 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::ADD), DestReg)
5210 .addReg(RegNo: DestReg, Flags: RegState::Kill)
5211 .addReg(RegNo: Acc, Flags: RegState::Kill)
5212 .setMIFlag(Flag);
5213 }
5214}
5215
5216ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
5217RISCVInstrInfo::getSerializableMachineMemOperandTargetFlags() const {
5218 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
5219 {{MONontemporalBit0, "riscv-nontemporal-domain-bit-0"},
5220 {MONontemporalBit1, "riscv-nontemporal-domain-bit-1"}};
5221 return ArrayRef(TargetFlags);
5222}
5223
5224unsigned RISCVInstrInfo::getTailDuplicateSize(CodeGenOptLevel OptLevel) const {
5225 return OptLevel >= CodeGenOptLevel::Aggressive
5226 ? STI.getTailDupAggressiveThreshold()
5227 : 2;
5228}
5229
5230bool RISCV::isRVVSpill(const MachineInstr &MI) {
5231 // RVV lacks any support for immediate addressing for stack addresses, so be
5232 // conservative.
5233 unsigned Opcode = MI.getOpcode();
5234 if (!RISCVVPseudosTable::getPseudoInfo(Pseudo: Opcode) &&
5235 !getLMULForRVVWholeLoadStore(Opcode) && !isRVVSpillForZvlsseg(Opcode))
5236 return false;
5237 return true;
5238}
5239
5240/// Return true if \p MI is a copy that will be lowered to one or more vmvNr.vs.
5241bool RISCV::isVectorCopy(const TargetRegisterInfo *TRI,
5242 const MachineInstr &MI) {
5243 return MI.isCopy() && MI.getOperand(i: 0).getReg().isPhysical() &&
5244 RISCVRegisterInfo::isRVVRegClass(
5245 RC: TRI->getMinimalPhysRegClass(Reg: MI.getOperand(i: 0).getReg()));
5246}
5247
5248std::optional<std::pair<unsigned, unsigned>>
5249RISCV::isRVVSpillForZvlsseg(unsigned Opcode) {
5250 switch (Opcode) {
5251 default:
5252 return std::nullopt;
5253 case RISCV::PseudoVSPILL2_M1:
5254 case RISCV::PseudoVRELOAD2_M1:
5255 return std::make_pair(x: 2u, y: 1u);
5256 case RISCV::PseudoVSPILL2_M2:
5257 case RISCV::PseudoVRELOAD2_M2:
5258 return std::make_pair(x: 2u, y: 2u);
5259 case RISCV::PseudoVSPILL2_M4:
5260 case RISCV::PseudoVRELOAD2_M4:
5261 return std::make_pair(x: 2u, y: 4u);
5262 case RISCV::PseudoVSPILL3_M1:
5263 case RISCV::PseudoVRELOAD3_M1:
5264 return std::make_pair(x: 3u, y: 1u);
5265 case RISCV::PseudoVSPILL3_M2:
5266 case RISCV::PseudoVRELOAD3_M2:
5267 return std::make_pair(x: 3u, y: 2u);
5268 case RISCV::PseudoVSPILL4_M1:
5269 case RISCV::PseudoVRELOAD4_M1:
5270 return std::make_pair(x: 4u, y: 1u);
5271 case RISCV::PseudoVSPILL4_M2:
5272 case RISCV::PseudoVRELOAD4_M2:
5273 return std::make_pair(x: 4u, y: 2u);
5274 case RISCV::PseudoVSPILL5_M1:
5275 case RISCV::PseudoVRELOAD5_M1:
5276 return std::make_pair(x: 5u, y: 1u);
5277 case RISCV::PseudoVSPILL6_M1:
5278 case RISCV::PseudoVRELOAD6_M1:
5279 return std::make_pair(x: 6u, y: 1u);
5280 case RISCV::PseudoVSPILL7_M1:
5281 case RISCV::PseudoVRELOAD7_M1:
5282 return std::make_pair(x: 7u, y: 1u);
5283 case RISCV::PseudoVSPILL8_M1:
5284 case RISCV::PseudoVRELOAD8_M1:
5285 return std::make_pair(x: 8u, y: 1u);
5286 }
5287}
5288
5289bool RISCV::hasEqualFRM(const MachineInstr &MI1, const MachineInstr &MI2) {
5290 int16_t MI1FrmOpIdx =
5291 RISCV::getNamedOperandIdx(Opcode: MI1.getOpcode(), Name: RISCV::OpName::frm);
5292 int16_t MI2FrmOpIdx =
5293 RISCV::getNamedOperandIdx(Opcode: MI2.getOpcode(), Name: RISCV::OpName::frm);
5294 if (MI1FrmOpIdx < 0 || MI2FrmOpIdx < 0)
5295 return false;
5296 MachineOperand FrmOp1 = MI1.getOperand(i: MI1FrmOpIdx);
5297 MachineOperand FrmOp2 = MI2.getOperand(i: MI2FrmOpIdx);
5298 return FrmOp1.getImm() == FrmOp2.getImm();
5299}
5300
5301std::optional<unsigned>
5302RISCV::getVectorLowDemandedScalarBits(unsigned Opcode, unsigned Log2SEW) {
5303 switch (Opcode) {
5304 default:
5305 return std::nullopt;
5306
5307 // 11.6. Vector Single-Width Shift Instructions
5308 case RISCV::VSLL_VX:
5309 case RISCV::VSRL_VX:
5310 case RISCV::VSRA_VX:
5311 // 12.4. Vector Single-Width Scaling Shift Instructions
5312 case RISCV::VSSRL_VX:
5313 case RISCV::VSSRA_VX:
5314 // Zvbb
5315 case RISCV::VROL_VX:
5316 case RISCV::VROR_VX:
5317 // Only the low lg2(SEW) bits of the shift-amount value are used.
5318 return Log2SEW;
5319
5320 // 11.7 Vector Narrowing Integer Right Shift Instructions
5321 case RISCV::VNSRL_WX:
5322 case RISCV::VNSRA_WX:
5323 // 12.5. Vector Narrowing Fixed-Point Clip Instructions
5324 case RISCV::VNCLIPU_WX:
5325 case RISCV::VNCLIP_WX:
5326 // Zvbb
5327 case RISCV::VWSLL_VX:
5328 // Only the low lg2(2*SEW) bits of the shift-amount value are used.
5329 return Log2SEW + 1;
5330
5331 // 11.1. Vector Single-Width Integer Add and Subtract
5332 case RISCV::VADD_VX:
5333 case RISCV::VSUB_VX:
5334 case RISCV::VRSUB_VX:
5335 // 11.2. Vector Widening Integer Add/Subtract
5336 case RISCV::VWADDU_VX:
5337 case RISCV::VWSUBU_VX:
5338 case RISCV::VWADD_VX:
5339 case RISCV::VWSUB_VX:
5340 case RISCV::VWADDU_WX:
5341 case RISCV::VWSUBU_WX:
5342 case RISCV::VWADD_WX:
5343 case RISCV::VWSUB_WX:
5344 // 11.4. Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions
5345 case RISCV::VADC_VXM:
5346 case RISCV::VADC_VIM:
5347 case RISCV::VMADC_VXM:
5348 case RISCV::VMADC_VIM:
5349 case RISCV::VMADC_VX:
5350 case RISCV::VSBC_VXM:
5351 case RISCV::VMSBC_VXM:
5352 case RISCV::VMSBC_VX:
5353 // 11.5 Vector Bitwise Logical Instructions
5354 case RISCV::VAND_VX:
5355 case RISCV::VOR_VX:
5356 case RISCV::VXOR_VX:
5357 // 11.8. Vector Integer Compare Instructions
5358 case RISCV::VMSEQ_VX:
5359 case RISCV::VMSNE_VX:
5360 case RISCV::VMSLTU_VX:
5361 case RISCV::VMSLT_VX:
5362 case RISCV::VMSLEU_VX:
5363 case RISCV::VMSLE_VX:
5364 case RISCV::VMSGTU_VX:
5365 case RISCV::VMSGT_VX:
5366 // 11.9. Vector Integer Min/Max Instructions
5367 case RISCV::VMINU_VX:
5368 case RISCV::VMIN_VX:
5369 case RISCV::VMAXU_VX:
5370 case RISCV::VMAX_VX:
5371 // 11.10. Vector Single-Width Integer Multiply Instructions
5372 case RISCV::VMUL_VX:
5373 case RISCV::VMULH_VX:
5374 case RISCV::VMULHU_VX:
5375 case RISCV::VMULHSU_VX:
5376 // 11.11. Vector Integer Divide Instructions
5377 case RISCV::VDIVU_VX:
5378 case RISCV::VDIV_VX:
5379 case RISCV::VREMU_VX:
5380 case RISCV::VREM_VX:
5381 // 11.12. Vector Widening Integer Multiply Instructions
5382 case RISCV::VWMUL_VX:
5383 case RISCV::VWMULU_VX:
5384 case RISCV::VWMULSU_VX:
5385 // 11.13. Vector Single-Width Integer Multiply-Add Instructions
5386 case RISCV::VMACC_VX:
5387 case RISCV::VNMSAC_VX:
5388 case RISCV::VMADD_VX:
5389 case RISCV::VNMSUB_VX:
5390 // 11.14. Vector Widening Integer Multiply-Add Instructions
5391 case RISCV::VWMACCU_VX:
5392 case RISCV::VWMACC_VX:
5393 case RISCV::VWMACCSU_VX:
5394 case RISCV::VWMACCUS_VX:
5395 // 11.15. Vector Integer Merge Instructions
5396 case RISCV::VMERGE_VXM:
5397 // 11.16. Vector Integer Move Instructions
5398 case RISCV::VMV_V_X:
5399 // 12.1. Vector Single-Width Saturating Add and Subtract
5400 case RISCV::VSADDU_VX:
5401 case RISCV::VSADD_VX:
5402 case RISCV::VSSUBU_VX:
5403 case RISCV::VSSUB_VX:
5404 // 12.2. Vector Single-Width Averaging Add and Subtract
5405 case RISCV::VAADDU_VX:
5406 case RISCV::VAADD_VX:
5407 case RISCV::VASUBU_VX:
5408 case RISCV::VASUB_VX:
5409 // 12.3. Vector Single-Width Fractional Multiply with Rounding and Saturation
5410 case RISCV::VSMUL_VX:
5411 // 16.1. Integer Scalar Move Instructions
5412 case RISCV::VMV_S_X:
5413 // Zvbb
5414 case RISCV::VANDN_VX:
5415 return 1U << Log2SEW;
5416 }
5417}
5418
5419unsigned RISCV::getRVVMCOpcode(unsigned RVVPseudoOpcode) {
5420 const RISCVVPseudosTable::PseudoInfo *RVV =
5421 RISCVVPseudosTable::getPseudoInfo(Pseudo: RVVPseudoOpcode);
5422 if (!RVV)
5423 return 0;
5424 return RVV->BaseInstr;
5425}
5426
5427unsigned RISCV::getDestLog2EEW(const MCInstrDesc &Desc, unsigned Log2SEW) {
5428 unsigned DestEEW =
5429 (Desc.TSFlags & RISCVII::DestEEWMask) >> RISCVII::DestEEWShift;
5430 // EEW = 1
5431 if (DestEEW == 0)
5432 return 0;
5433 // EEW = SEW * n
5434 unsigned Scaled = Log2SEW + (DestEEW - 1);
5435 assert(Scaled >= 3 && Scaled <= 6);
5436 return Scaled;
5437}
5438
5439static std::optional<int64_t> getEffectiveImm(const MachineRegisterInfo &MRI,
5440 const MachineOperand &MO) {
5441 assert(MO.isImm() || MO.getReg().isVirtual());
5442 if (MO.isImm())
5443 return MO.getImm();
5444 const MachineInstr *Def = MRI.getVRegDef(Reg: MO.getReg());
5445 int64_t Imm;
5446 if (isLoadImm(MI: Def, Imm))
5447 return Imm;
5448 return std::nullopt;
5449}
5450
5451/// Given two VL operands, do we know that LHS <= RHS? Must be used in SSA form.
5452bool RISCV::isVLKnownLE(const MachineRegisterInfo &MRI,
5453 const MachineOperand &LHS, const MachineOperand &RHS) {
5454 assert((LHS.isImm() || MRI.isSSA()) && (RHS.isImm() || MRI.isSSA()));
5455 if (LHS.isReg() && RHS.isReg() && LHS.getReg().isVirtual() &&
5456 LHS.getReg() == RHS.getReg())
5457 return true;
5458 if (RHS.isImm() && RHS.getImm() == RISCV::VLMaxSentinel)
5459 return true;
5460 if (LHS.isImm() && LHS.getImm() == 0)
5461 return true;
5462 if (LHS.isImm() && LHS.getImm() == RISCV::VLMaxSentinel)
5463 return false;
5464 std::optional<int64_t> LHSImm = getEffectiveImm(MRI, MO: LHS),
5465 RHSImm = getEffectiveImm(MRI, MO: RHS);
5466 if (!LHSImm || !RHSImm)
5467 return false;
5468 return LHSImm <= RHSImm;
5469}
5470
5471namespace {
5472class RISCVPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
5473 const MachineInstr *LHS;
5474 const MachineInstr *RHS;
5475 SmallVector<MachineOperand, 3> Cond;
5476
5477public:
5478 RISCVPipelinerLoopInfo(const MachineInstr *LHS, const MachineInstr *RHS,
5479 const SmallVectorImpl<MachineOperand> &Cond)
5480 : LHS(LHS), RHS(RHS), Cond(Cond.begin(), Cond.end()) {}
5481
5482 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
5483 // Make the instructions for loop control be placed in stage 0.
5484 // The predecessors of LHS/RHS are considered by the caller.
5485 if (LHS && MI == LHS)
5486 return true;
5487 if (RHS && MI == RHS)
5488 return true;
5489 return false;
5490 }
5491
5492 std::optional<bool> createTripCountGreaterCondition(
5493 int TC, MachineBasicBlock &MBB,
5494 SmallVectorImpl<MachineOperand> &CondParam) override {
5495 // A branch instruction will be inserted as "if (Cond) goto epilogue".
5496 // Cond is normalized for such use.
5497 // The predecessors of the branch are assumed to have already been inserted.
5498 CondParam = Cond;
5499 return {};
5500 }
5501
5502 void setPreheader(MachineBasicBlock *NewPreheader) override {}
5503
5504 void adjustTripCount(int TripCountAdjust) override {}
5505};
5506} // namespace
5507
5508std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
5509RISCVInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
5510 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
5511 SmallVector<MachineOperand, 4> Cond;
5512 if (analyzeBranch(MBB&: *LoopBB, TBB, FBB, Cond, /*AllowModify=*/false))
5513 return nullptr;
5514
5515 // Infinite loops are not supported
5516 if (TBB == LoopBB && FBB == LoopBB)
5517 return nullptr;
5518
5519 // Must be conditional branch
5520 if (FBB == nullptr)
5521 return nullptr;
5522
5523 assert((TBB == LoopBB || FBB == LoopBB) &&
5524 "The Loop must be a single-basic-block loop");
5525
5526 // Normalization for createTripCountGreaterCondition()
5527 if (TBB == LoopBB)
5528 reverseBranchCondition(Cond);
5529
5530 const MachineRegisterInfo &MRI = LoopBB->getParent()->getRegInfo();
5531 auto FindRegDef = [&MRI](MachineOperand &Op) -> const MachineInstr * {
5532 if (!Op.isReg())
5533 return nullptr;
5534 Register Reg = Op.getReg();
5535 if (!Reg.isVirtual())
5536 return nullptr;
5537 return MRI.getVRegDef(Reg);
5538 };
5539
5540 const MachineInstr *LHS = FindRegDef(Cond[1]);
5541 const MachineInstr *RHS = FindRegDef(Cond[2]);
5542 if (LHS && LHS->isPHI())
5543 return nullptr;
5544 if (RHS && RHS->isPHI())
5545 return nullptr;
5546
5547 return std::make_unique<RISCVPipelinerLoopInfo>(args&: LHS, args&: RHS, args&: Cond);
5548}
5549
5550// FIXME: We should remove this if we have a default generic scheduling model.
5551bool RISCVInstrInfo::isHighLatencyDef(int Opc) const {
5552 unsigned RVVMCOpcode = RISCV::getRVVMCOpcode(RVVPseudoOpcode: Opc);
5553 Opc = RVVMCOpcode ? RVVMCOpcode : Opc;
5554 switch (Opc) {
5555 default:
5556 return false;
5557 // Integer div/rem.
5558 case RISCV::DIV:
5559 case RISCV::DIVW:
5560 case RISCV::DIVU:
5561 case RISCV::DIVUW:
5562 case RISCV::REM:
5563 case RISCV::REMW:
5564 case RISCV::REMU:
5565 case RISCV::REMUW:
5566 // Floating-point div/sqrt.
5567 case RISCV::FDIV_H:
5568 case RISCV::FDIV_S:
5569 case RISCV::FDIV_D:
5570 case RISCV::FDIV_H_INX:
5571 case RISCV::FDIV_S_INX:
5572 case RISCV::FDIV_D_INX:
5573 case RISCV::FDIV_D_IN32X:
5574 case RISCV::FSQRT_H:
5575 case RISCV::FSQRT_S:
5576 case RISCV::FSQRT_D:
5577 case RISCV::FSQRT_H_INX:
5578 case RISCV::FSQRT_S_INX:
5579 case RISCV::FSQRT_D_INX:
5580 case RISCV::FSQRT_D_IN32X:
5581 // Vector integer div/rem
5582 case RISCV::VDIV_VV:
5583 case RISCV::VDIV_VX:
5584 case RISCV::VDIVU_VV:
5585 case RISCV::VDIVU_VX:
5586 case RISCV::VREM_VV:
5587 case RISCV::VREM_VX:
5588 case RISCV::VREMU_VV:
5589 case RISCV::VREMU_VX:
5590 // Vector floating-point div/sqrt.
5591 case RISCV::VFDIV_VV:
5592 case RISCV::VFDIV_VF:
5593 case RISCV::VFRDIV_VF:
5594 case RISCV::VFSQRT_V:
5595 case RISCV::VFRSQRT7_V:
5596 return true;
5597 }
5598}
5599
5600bool RISCVInstrInfo::isVRegCopy(const MachineInstr *MI, unsigned LMul) const {
5601 if (MI->getOpcode() != TargetOpcode::COPY)
5602 return false;
5603 const MachineRegisterInfo &MRI = MI->getMF()->getRegInfo();
5604 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
5605
5606 Register DstReg = MI->getOperand(i: 0).getReg();
5607 const TargetRegisterClass *RC = DstReg.isVirtual()
5608 ? MRI.getRegClass(Reg: DstReg)
5609 : TRI->getMinimalPhysRegClass(Reg: DstReg);
5610
5611 if (!RISCVRegisterInfo::isRVVRegClass(RC))
5612 return false;
5613
5614 if (!LMul)
5615 return true;
5616
5617 // TODO: Perhaps we could distinguish segment register classes (e.g. VRN3M2)
5618 // in the future.
5619 auto [RCLMul, RCFractional] =
5620 RISCVVType::decodeVLMUL(VLMul: RISCVRI::getLMul(TSFlags: RC->TSFlags));
5621 return (!RCFractional && LMul == RCLMul) || (RCFractional && LMul == 1);
5622}
5623
5624bool RISCVInstrInfo::requiresNTLHint(const MachineInstr &MI) const {
5625 if (MI.memoperands_empty())
5626 return false;
5627
5628 MachineMemOperand *MMO = *(MI.memoperands_begin());
5629 if (!MMO->isNonTemporal())
5630 return false;
5631
5632 return true;
5633}
5634
5635bool RISCVInstrInfo::isSafeToMove(const MachineInstr &From,
5636 const MachineBasicBlock::iterator &To) {
5637 assert(To == From.getParent()->end() || From.getParent() == To->getParent());
5638 SmallVector<Register> PhysUses, PhysDefs;
5639 for (const MachineOperand &MO : From.all_uses())
5640 if (MO.getReg().isPhysical())
5641 PhysUses.push_back(Elt: MO.getReg());
5642 for (const MachineOperand &MO : From.all_defs())
5643 if (MO.getReg().isPhysical())
5644 PhysDefs.push_back(Elt: MO.getReg());
5645 bool SawStore = false;
5646 for (auto II = std::next(x: From.getIterator()); II != To; II++) {
5647 for (Register PhysReg : PhysUses)
5648 if (II->definesRegister(Reg: PhysReg, TRI: nullptr))
5649 return false;
5650 for (Register PhysReg : PhysDefs)
5651 if (II->definesRegister(Reg: PhysReg, TRI: nullptr) ||
5652 II->readsRegister(Reg: PhysReg, TRI: nullptr))
5653 return false;
5654 II->isSafeToMove(SawStore);
5655 if (SawStore)
5656 break;
5657 }
5658 return From.isSafeToMove(SawStore);
5659}
5660