1//===-- RISCVInstrInfo.cpp - RISC-V Instruction Information -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the RISC-V implementation of the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RISCVInstrInfo.h"
14#include "MCTargetDesc/RISCVBaseInfo.h"
15#include "MCTargetDesc/RISCVMatInt.h"
16#include "RISCV.h"
17#include "RISCVMachineFunctionInfo.h"
18#include "RISCVSubtarget.h"
19#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/ADT/Statistic.h"
22#include "llvm/Analysis/MemoryLocation.h"
23#include "llvm/Analysis/ValueTracking.h"
24#include "llvm/CodeGen/LiveIntervals.h"
25#include "llvm/CodeGen/LiveVariables.h"
26#include "llvm/CodeGen/MachineCombinerPattern.h"
27#include "llvm/CodeGen/MachineInstrBuilder.h"
28#include "llvm/CodeGen/MachineRegisterInfo.h"
29#include "llvm/CodeGen/MachineTraceMetrics.h"
30#include "llvm/CodeGen/RegisterScavenging.h"
31#include "llvm/CodeGen/StackMaps.h"
32#include "llvm/IR/DebugInfoMetadata.h"
33#include "llvm/IR/Module.h"
34#include "llvm/MC/MCDwarf.h"
35#include "llvm/MC/MCInstBuilder.h"
36#include "llvm/MC/TargetRegistry.h"
37#include "llvm/Support/ErrorHandling.h"
38
39using namespace llvm;
40
41#define GEN_CHECK_COMPRESS_INSTR
42#include "RISCVGenCompressInstEmitter.inc"
43
44#define GET_INSTRINFO_CTOR_DTOR
45#include "RISCVGenInstrInfo.inc"
46
47#define DEBUG_TYPE "riscv-instr-info"
48STATISTIC(NumVRegSpilled,
49 "Number of registers within vector register groups spilled");
50STATISTIC(NumVRegReloaded,
51 "Number of registers within vector register groups reloaded");
52
53static cl::opt<bool> PreferWholeRegisterMove(
54 "riscv-prefer-whole-register-move", cl::init(Val: false), cl::Hidden,
55 cl::desc("Prefer whole register move for vector registers."));
56
57static cl::opt<MachineTraceStrategy> ForceMachineCombinerStrategy(
58 "riscv-force-machine-combiner-strategy", cl::Hidden,
59 cl::desc("Force machine combiner to use a specific strategy for machine "
60 "trace metrics evaluation."),
61 cl::init(Val: MachineTraceStrategy::TS_NumStrategies),
62 cl::values(clEnumValN(MachineTraceStrategy::TS_Local, "local",
63 "Local strategy."),
64 clEnumValN(MachineTraceStrategy::TS_MinInstrCount, "min-instr",
65 "MinInstrCount strategy.")));
66
67static cl::opt<bool> OutlinerEnableRegSave(
68 "riscv-outliner-regsave", cl::init(Val: true), cl::Hidden,
69 cl::desc("Enable RegSave strategy in machine outliner (save X5 to a "
70 "temporary register when X5 is live across outlined calls)."));
71
72namespace llvm::RISCVVPseudosTable {
73
74using namespace RISCV;
75
76#define GET_RISCVVPseudosTable_IMPL
77#include "RISCVGenSearchableTables.inc"
78
79} // namespace llvm::RISCVVPseudosTable
80
81namespace llvm::RISCV {
82
83#define GET_RISCVMaskedPseudosTable_IMPL
84#include "RISCVGenSearchableTables.inc"
85
86} // end namespace llvm::RISCV
87
88RISCVInstrInfo::RISCVInstrInfo(const RISCVSubtarget &STI)
89 : RISCVGenInstrInfo(STI, RegInfo, RISCV::ADJCALLSTACKDOWN,
90 RISCV::ADJCALLSTACKUP),
91 RegInfo(STI.getHwMode()), STI(STI) {}
92
93#define GET_INSTRINFO_HELPERS
94#include "RISCVGenInstrInfo.inc"
95
96MCInst RISCVInstrInfo::getNop() const {
97 if (STI.hasStdExtZca())
98 return MCInstBuilder(RISCV::C_NOP);
99 return MCInstBuilder(RISCV::ADDI)
100 .addReg(Reg: RISCV::X0)
101 .addReg(Reg: RISCV::X0)
102 .addImm(Val: 0);
103}
104
105Register RISCVInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
106 int &FrameIndex) const {
107 TypeSize Dummy = TypeSize::getZero();
108 return isLoadFromStackSlot(MI, FrameIndex, MemBytes&: Dummy);
109}
110
111static std::optional<unsigned> getLMULForRVVWholeLoadStore(unsigned Opcode) {
112 switch (Opcode) {
113 default:
114 return std::nullopt;
115 case RISCV::VS1R_V:
116 case RISCV::VL1RE8_V:
117 case RISCV::VL1RE16_V:
118 case RISCV::VL1RE32_V:
119 case RISCV::VL1RE64_V:
120 return 1;
121 case RISCV::VS2R_V:
122 case RISCV::VL2RE8_V:
123 case RISCV::VL2RE16_V:
124 case RISCV::VL2RE32_V:
125 case RISCV::VL2RE64_V:
126 return 2;
127 case RISCV::VS4R_V:
128 case RISCV::VL4RE8_V:
129 case RISCV::VL4RE16_V:
130 case RISCV::VL4RE32_V:
131 case RISCV::VL4RE64_V:
132 return 4;
133 case RISCV::VS8R_V:
134 case RISCV::VL8RE8_V:
135 case RISCV::VL8RE16_V:
136 case RISCV::VL8RE32_V:
137 case RISCV::VL8RE64_V:
138 return 8;
139 }
140}
141
142Register RISCVInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
143 int &FrameIndex,
144 TypeSize &MemBytes) const {
145 switch (MI.getOpcode()) {
146 default:
147 return 0;
148 case RISCV::LB:
149 case RISCV::LBU:
150 MemBytes = TypeSize::getFixed(ExactSize: 1);
151 break;
152 case RISCV::LH:
153 case RISCV::LH_INX:
154 case RISCV::LHU:
155 case RISCV::FLH:
156 MemBytes = TypeSize::getFixed(ExactSize: 2);
157 break;
158 case RISCV::LW:
159 case RISCV::LW_INX:
160 case RISCV::FLW:
161 case RISCV::LWU:
162 MemBytes = TypeSize::getFixed(ExactSize: 4);
163 break;
164 case RISCV::LD:
165 case RISCV::LD_RV32:
166 case RISCV::FLD:
167 MemBytes = TypeSize::getFixed(ExactSize: 8);
168 break;
169 case RISCV::VL1RE8_V:
170 case RISCV::VL2RE8_V:
171 case RISCV::VL4RE8_V:
172 case RISCV::VL8RE8_V:
173 if (!MI.getOperand(i: 1).isFI())
174 return Register();
175 FrameIndex = MI.getOperand(i: 1).getIndex();
176 unsigned LMUL = *getLMULForRVVWholeLoadStore(Opcode: MI.getOpcode());
177 MemBytes = TypeSize::getScalable(MinimumSize: RISCV::RVVBytesPerBlock * LMUL);
178 return MI.getOperand(i: 0).getReg();
179 }
180
181 if (MI.getOperand(i: 1).isFI() && MI.getOperand(i: 2).isImm() &&
182 MI.getOperand(i: 2).getImm() == 0) {
183 FrameIndex = MI.getOperand(i: 1).getIndex();
184 return MI.getOperand(i: 0).getReg();
185 }
186
187 return 0;
188}
189
190Register RISCVInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
191 int &FrameIndex) const {
192 TypeSize Dummy = TypeSize::getZero();
193 return isStoreToStackSlot(MI, FrameIndex, MemBytes&: Dummy);
194}
195
196Register RISCVInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
197 int &FrameIndex,
198 TypeSize &MemBytes) const {
199 switch (MI.getOpcode()) {
200 default:
201 return 0;
202 case RISCV::SB:
203 MemBytes = TypeSize::getFixed(ExactSize: 1);
204 break;
205 case RISCV::SH:
206 case RISCV::SH_INX:
207 case RISCV::FSH:
208 MemBytes = TypeSize::getFixed(ExactSize: 2);
209 break;
210 case RISCV::SW:
211 case RISCV::SW_INX:
212 case RISCV::FSW:
213 MemBytes = TypeSize::getFixed(ExactSize: 4);
214 break;
215 case RISCV::SD:
216 case RISCV::SD_RV32:
217 case RISCV::FSD:
218 MemBytes = TypeSize::getFixed(ExactSize: 8);
219 break;
220 case RISCV::VS1R_V:
221 case RISCV::VS2R_V:
222 case RISCV::VS4R_V:
223 case RISCV::VS8R_V:
224 if (!MI.getOperand(i: 1).isFI())
225 return Register();
226 FrameIndex = MI.getOperand(i: 1).getIndex();
227 unsigned LMUL = *getLMULForRVVWholeLoadStore(Opcode: MI.getOpcode());
228 MemBytes = TypeSize::getScalable(MinimumSize: RISCV::RVVBytesPerBlock * LMUL);
229 return MI.getOperand(i: 0).getReg();
230 }
231
232 if (MI.getOperand(i: 1).isFI() && MI.getOperand(i: 2).isImm() &&
233 MI.getOperand(i: 2).getImm() == 0) {
234 FrameIndex = MI.getOperand(i: 1).getIndex();
235 return MI.getOperand(i: 0).getReg();
236 }
237
238 return 0;
239}
240
241bool RISCVInstrInfo::isReMaterializableImpl(
242 const MachineInstr &MI) const {
243 switch (RISCV::getRVVMCOpcode(RVVPseudoOpcode: MI.getOpcode())) {
244 case RISCV::VMV_V_X:
245 case RISCV::VFMV_V_F:
246 case RISCV::VMV_V_I:
247 case RISCV::VMV_S_X:
248 case RISCV::VFMV_S_F:
249 case RISCV::VID_V:
250 return MI.getOperand(i: 1).isUndef();
251 default:
252 return TargetInstrInfo::isReMaterializableImpl(MI);
253 }
254}
255
256static bool forwardCopyWillClobberTuple(unsigned DstReg, unsigned SrcReg,
257 unsigned NumRegs) {
258 return DstReg > SrcReg && (DstReg - SrcReg) < NumRegs;
259}
260
261static bool isConvertibleToVMV_V_V(const RISCVSubtarget &STI,
262 const MachineBasicBlock &MBB,
263 MachineBasicBlock::const_iterator MBBI,
264 MachineBasicBlock::const_iterator &DefMBBI,
265 RISCVVType::VLMUL LMul) {
266 if (PreferWholeRegisterMove)
267 return false;
268
269 assert(MBBI->getOpcode() == TargetOpcode::COPY &&
270 "Unexpected COPY instruction.");
271 Register SrcReg = MBBI->getOperand(i: 1).getReg();
272 const TargetRegisterInfo *TRI = STI.getRegisterInfo();
273
274 bool FoundDef = false;
275 bool FirstVSetVLI = false;
276 unsigned FirstSEW = 0;
277 while (MBBI != MBB.begin()) {
278 --MBBI;
279 if (MBBI->isMetaInstruction())
280 continue;
281
282 if (RISCVInstrInfo::isVectorConfigInstr(MI: *MBBI)) {
283 // There is a vsetvli between COPY and source define instruction.
284 // vy = def_vop ... (producing instruction)
285 // ...
286 // vsetvli
287 // ...
288 // vx = COPY vy
289 if (!FoundDef) {
290 if (!FirstVSetVLI) {
291 FirstVSetVLI = true;
292 unsigned FirstVType = MBBI->getOperand(i: 2).getImm();
293 RISCVVType::VLMUL FirstLMul = RISCVVType::getVLMUL(VType: FirstVType);
294 FirstSEW = RISCVVType::getSEW(VType: FirstVType);
295 // The first encountered vsetvli must have the same lmul as the
296 // register class of COPY.
297 if (FirstLMul != LMul)
298 return false;
299 }
300 // Only permit `vsetvli x0, x0, vtype` between COPY and the source
301 // define instruction.
302 if (!RISCVInstrInfo::isVLPreservingConfig(MI: *MBBI))
303 return false;
304 continue;
305 }
306
307 // MBBI is the first vsetvli before the producing instruction.
308 unsigned VType = MBBI->getOperand(i: 2).getImm();
309 // If there is a vsetvli between COPY and the producing instruction.
310 if (FirstVSetVLI) {
311 // If SEW is different, return false.
312 if (RISCVVType::getSEW(VType) != FirstSEW)
313 return false;
314 }
315
316 // If the vsetvli is tail undisturbed, keep the whole register move.
317 if (!RISCVVType::isTailAgnostic(VType))
318 return false;
319
320 // The checking is conservative. We only have register classes for
321 // LMUL = 1/2/4/8. We should be able to convert vmv1r.v to vmv.v.v
322 // for fractional LMUL operations. However, we could not use the vsetvli
323 // lmul for widening operations. The result of widening operation is
324 // 2 x LMUL.
325 return LMul == RISCVVType::getVLMUL(VType);
326 } else if (MBBI->isInlineAsm() || MBBI->isCall()) {
327 return false;
328 } else if (MBBI->getNumDefs()) {
329 // Check all the instructions which will change VL.
330 // For example, vleff has implicit def VL.
331 if (MBBI->modifiesRegister(Reg: RISCV::VL, /*TRI=*/nullptr))
332 return false;
333
334 // Only converting whole register copies to vmv.v.v when the defining
335 // value appears in the explicit operands.
336 for (const MachineOperand &MO : MBBI->explicit_operands()) {
337 if (!MO.isReg() || !MO.isDef())
338 continue;
339 if (!FoundDef && TRI->regsOverlap(RegA: MO.getReg(), RegB: SrcReg)) {
340 // We only permit the source of COPY has the same LMUL as the defined
341 // operand.
342 // There are cases we need to keep the whole register copy if the LMUL
343 // is different.
344 // For example,
345 // $x0 = PseudoVSETIVLI 4, 73 // vsetivli zero, 4, e16,m2,ta,m
346 // $v28m4 = PseudoVWADD_VV_M2 $v26m2, $v8m2
347 // # The COPY may be created by vlmul_trunc intrinsic.
348 // $v26m2 = COPY renamable $v28m2, implicit killed $v28m4
349 //
350 // After widening, the valid value will be 4 x e32 elements. If we
351 // convert the COPY to vmv.v.v, it will only copy 4 x e16 elements.
352 // FIXME: The COPY of subregister of Zvlsseg register will not be able
353 // to convert to vmv.v.[v|i] under the constraint.
354 if (MO.getReg() != SrcReg)
355 return false;
356
357 // In widening reduction instructions with LMUL_1 input vector case,
358 // only checking the LMUL is insufficient due to reduction result is
359 // always LMUL_1.
360 // For example,
361 // $x11 = PseudoVSETIVLI 1, 64 // vsetivli a1, 1, e8, m1, ta, mu
362 // $v8m1 = PseudoVWREDSUM_VS_M1 $v26, $v27
363 // $v26 = COPY killed renamable $v8
364 // After widening, The valid value will be 1 x e16 elements. If we
365 // convert the COPY to vmv.v.v, it will only copy 1 x e8 elements.
366 uint64_t TSFlags = MBBI->getDesc().TSFlags;
367 if (RISCVII::isRVVWideningReduction(TSFlags))
368 return false;
369
370 // If the producing instruction does not depend on vsetvli, do not
371 // convert COPY to vmv.v.v. For example, VL1R_V or PseudoVRELOAD.
372 if (!RISCVII::hasSEWOp(TSFlags) || !RISCVII::hasVLOp(TSFlags))
373 return false;
374
375 // Found the definition.
376 FoundDef = true;
377 DefMBBI = MBBI;
378 break;
379 }
380 }
381 }
382 }
383
384 return false;
385}
386
387void RISCVInstrInfo::copyPhysRegVector(
388 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
389 const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc,
390 const TargetRegisterClass *RegClass) const {
391 const RISCVRegisterInfo *TRI = STI.getRegisterInfo();
392 RISCVVType::VLMUL LMul = RISCVRI::getLMul(TSFlags: RegClass->TSFlags);
393 unsigned NF = RISCVRI::getNF(TSFlags: RegClass->TSFlags);
394
395 uint16_t SrcEncoding = TRI->getEncodingValue(Reg: SrcReg);
396 uint16_t DstEncoding = TRI->getEncodingValue(Reg: DstReg);
397 auto [LMulVal, Fractional] = RISCVVType::decodeVLMUL(VLMul: LMul);
398 assert(!Fractional && "It is impossible be fractional lmul here.");
399 unsigned NumRegs = NF * LMulVal;
400 bool ReversedCopy =
401 forwardCopyWillClobberTuple(DstReg: DstEncoding, SrcReg: SrcEncoding, NumRegs);
402 if (ReversedCopy) {
403 // If the src and dest overlap when copying a tuple, we need to copy the
404 // registers in reverse.
405 SrcEncoding += NumRegs - 1;
406 DstEncoding += NumRegs - 1;
407 }
408
409 unsigned I = 0;
410 auto GetCopyInfo = [&](uint16_t SrcEncoding, uint16_t DstEncoding)
411 -> std::tuple<RISCVVType::VLMUL, const TargetRegisterClass &, unsigned,
412 unsigned, unsigned> {
413 if (ReversedCopy) {
414 // For reversed copying, if there are enough aligned registers(8/4/2), we
415 // can do a larger copy(LMUL8/4/2).
416 // Besides, we have already known that DstEncoding is larger than
417 // SrcEncoding in forwardCopyWillClobberTuple, so the difference between
418 // DstEncoding and SrcEncoding should be >= LMUL value we try to use to
419 // avoid clobbering.
420 uint16_t Diff = DstEncoding - SrcEncoding;
421 if (I + 8 <= NumRegs && Diff >= 8 && SrcEncoding % 8 == 7 &&
422 DstEncoding % 8 == 7)
423 return {RISCVVType::LMUL_8, RISCV::VRM8RegClass, RISCV::VMV8R_V,
424 RISCV::PseudoVMV_V_V_M8, RISCV::PseudoVMV_V_I_M8};
425 if (I + 4 <= NumRegs && Diff >= 4 && SrcEncoding % 4 == 3 &&
426 DstEncoding % 4 == 3)
427 return {RISCVVType::LMUL_4, RISCV::VRM4RegClass, RISCV::VMV4R_V,
428 RISCV::PseudoVMV_V_V_M4, RISCV::PseudoVMV_V_I_M4};
429 if (I + 2 <= NumRegs && Diff >= 2 && SrcEncoding % 2 == 1 &&
430 DstEncoding % 2 == 1)
431 return {RISCVVType::LMUL_2, RISCV::VRM2RegClass, RISCV::VMV2R_V,
432 RISCV::PseudoVMV_V_V_M2, RISCV::PseudoVMV_V_I_M2};
433 // Or we should do LMUL1 copying.
434 return {RISCVVType::LMUL_1, RISCV::VRRegClass, RISCV::VMV1R_V,
435 RISCV::PseudoVMV_V_V_M1, RISCV::PseudoVMV_V_I_M1};
436 }
437
438 // For forward copying, if source register encoding and destination register
439 // encoding are aligned to 8/4/2, we can do a LMUL8/4/2 copying.
440 if (I + 8 <= NumRegs && SrcEncoding % 8 == 0 && DstEncoding % 8 == 0)
441 return {RISCVVType::LMUL_8, RISCV::VRM8RegClass, RISCV::VMV8R_V,
442 RISCV::PseudoVMV_V_V_M8, RISCV::PseudoVMV_V_I_M8};
443 if (I + 4 <= NumRegs && SrcEncoding % 4 == 0 && DstEncoding % 4 == 0)
444 return {RISCVVType::LMUL_4, RISCV::VRM4RegClass, RISCV::VMV4R_V,
445 RISCV::PseudoVMV_V_V_M4, RISCV::PseudoVMV_V_I_M4};
446 if (I + 2 <= NumRegs && SrcEncoding % 2 == 0 && DstEncoding % 2 == 0)
447 return {RISCVVType::LMUL_2, RISCV::VRM2RegClass, RISCV::VMV2R_V,
448 RISCV::PseudoVMV_V_V_M2, RISCV::PseudoVMV_V_I_M2};
449 // Or we should do LMUL1 copying.
450 return {RISCVVType::LMUL_1, RISCV::VRRegClass, RISCV::VMV1R_V,
451 RISCV::PseudoVMV_V_V_M1, RISCV::PseudoVMV_V_I_M1};
452 };
453
454 while (I != NumRegs) {
455 // For non-segment copying, we only do this once as the registers are always
456 // aligned.
457 // For segment copying, we may do this several times. If the registers are
458 // aligned to larger LMUL, we can eliminate some copyings.
459 auto [LMulCopied, RegClass, Opc, VVOpc, VIOpc] =
460 GetCopyInfo(SrcEncoding, DstEncoding);
461 auto [NumCopied, _] = RISCVVType::decodeVLMUL(VLMul: LMulCopied);
462
463 MachineBasicBlock::const_iterator DefMBBI;
464 if (LMul == LMulCopied &&
465 isConvertibleToVMV_V_V(STI, MBB, MBBI, DefMBBI, LMul)) {
466 Opc = VVOpc;
467 if (DefMBBI->getOpcode() == VIOpc)
468 Opc = VIOpc;
469 }
470
471 // Emit actual copying.
472 // For reversed copying, the encoding should be decreased.
473 MCRegister ActualSrcReg = TRI->findVRegWithEncoding(
474 RegClass, Encoding: ReversedCopy ? (SrcEncoding - NumCopied + 1) : SrcEncoding);
475 MCRegister ActualDstReg = TRI->findVRegWithEncoding(
476 RegClass, Encoding: ReversedCopy ? (DstEncoding - NumCopied + 1) : DstEncoding);
477
478 auto MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: Opc), DestReg: ActualDstReg);
479 bool UseVMV_V_I = RISCV::getRVVMCOpcode(RVVPseudoOpcode: Opc) == RISCV::VMV_V_I;
480 bool UseVMV = UseVMV_V_I || RISCV::getRVVMCOpcode(RVVPseudoOpcode: Opc) == RISCV::VMV_V_V;
481 if (UseVMV)
482 MIB.addReg(RegNo: ActualDstReg, Flags: RegState::Undef);
483 if (UseVMV_V_I)
484 MIB = MIB.add(MO: DefMBBI->getOperand(i: 2));
485 else
486 MIB = MIB.addReg(RegNo: ActualSrcReg, Flags: getKillRegState(B: KillSrc));
487 if (UseVMV) {
488 const MCInstrDesc &Desc = DefMBBI->getDesc();
489 MIB.add(MO: DefMBBI->getOperand(i: RISCVII::getVLOpNum(Desc))); // AVL
490 unsigned Log2SEW =
491 DefMBBI->getOperand(i: RISCVII::getSEWOpNum(Desc)).getImm();
492 MIB.addImm(Val: Log2SEW ? Log2SEW : 3); // SEW
493 MIB.addImm(Val: 0); // tu, mu
494 MIB.addReg(RegNo: RISCV::VL, Flags: RegState::Implicit);
495 MIB.addReg(RegNo: RISCV::VTYPE, Flags: RegState::Implicit);
496 }
497 // Add an implicit read of the original source to silence the verifier
498 // in the cases where some of the smaller VRs we're copying from might be
499 // undef, caused by the fact that the original, larger source VR might not
500 // be fully initialized at the time this COPY happens.
501 MIB.addReg(RegNo: SrcReg, Flags: RegState::Implicit);
502
503 // If we are copying reversely, we should decrease the encoding.
504 SrcEncoding += (ReversedCopy ? -NumCopied : NumCopied);
505 DstEncoding += (ReversedCopy ? -NumCopied : NumCopied);
506 I += NumCopied;
507 }
508}
509
510void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
511 MachineBasicBlock::iterator MBBI,
512 const DebugLoc &DL, Register DstReg,
513 Register SrcReg, bool KillSrc,
514 bool RenamableDest, bool RenamableSrc) const {
515 const TargetRegisterInfo *TRI = STI.getRegisterInfo();
516 RegState KillFlag = getKillRegState(B: KillSrc);
517
518 if (RISCV::GPRRegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
519 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::ADDI), DestReg: DstReg)
520 .addReg(RegNo: SrcReg, Flags: KillFlag | getRenamableRegState(B: RenamableSrc))
521 .addImm(Val: 0);
522 return;
523 }
524
525 // Extracting from X0_Pair may create copies from DUMMY_REG_PAIR_WITH_X0.
526 if (SrcReg == RISCV::DUMMY_REG_PAIR_WITH_X0 &&
527 RISCV::GPRRegClass.contains(Reg: DstReg)) {
528 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::ADDI), DestReg: DstReg)
529 .addReg(RegNo: RISCV::X0)
530 .addImm(Val: 0);
531 return;
532 }
533
534 if (RISCV::GPRF16RegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
535 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::PseudoMV_FPR16INX), DestReg: DstReg)
536 .addReg(RegNo: SrcReg, Flags: KillFlag | getRenamableRegState(B: RenamableSrc));
537 return;
538 }
539
540 if (RISCV::GPRF32RegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
541 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::PseudoMV_FPR32INX), DestReg: DstReg)
542 .addReg(RegNo: SrcReg, Flags: KillFlag | getRenamableRegState(B: RenamableSrc));
543 return;
544 }
545
546 if (RISCV::GPRPairRegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
547 if (!STI.is64Bit()) {
548 if (STI.hasStdExtZdinx()) {
549 // On RV32_Zdinx, FMV.D will move a pair of registers to another pair of
550 // registers, in one instruction.
551 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FSGNJ_D_IN32X), DestReg: DstReg)
552 .addReg(RegNo: SrcReg, Flags: getRenamableRegState(B: RenamableSrc))
553 .addReg(RegNo: SrcReg, Flags: KillFlag | getRenamableRegState(B: RenamableSrc));
554 return;
555 }
556
557 if (STI.hasStdExtP()) {
558 // On RV32P, `padd.dw` is a GPR Pair Add
559 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::PADD_DW), DestReg: DstReg)
560 .addReg(RegNo: RISCV::X0_Pair)
561 .addReg(RegNo: SrcReg, Flags: KillFlag | getRenamableRegState(B: RenamableSrc));
562 return;
563 }
564 }
565
566 MCRegister EvenReg = TRI->getSubReg(Reg: SrcReg, Idx: RISCV::sub_gpr_even);
567 MCRegister OddReg = TRI->getSubReg(Reg: SrcReg, Idx: RISCV::sub_gpr_odd);
568 // We need to correct the odd register of X0_Pair.
569 if (OddReg == RISCV::DUMMY_REG_PAIR_WITH_X0)
570 OddReg = RISCV::X0;
571 assert(DstReg != RISCV::X0_Pair && "Cannot write to X0_Pair");
572
573 // Emit an ADDI for both parts of GPRPair.
574 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::ADDI),
575 DestReg: TRI->getSubReg(Reg: DstReg, Idx: RISCV::sub_gpr_even))
576 .addReg(RegNo: EvenReg, Flags: KillFlag)
577 .addImm(Val: 0);
578 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::ADDI),
579 DestReg: TRI->getSubReg(Reg: DstReg, Idx: RISCV::sub_gpr_odd))
580 .addReg(RegNo: OddReg, Flags: KillFlag)
581 .addImm(Val: 0);
582 return;
583 }
584
585 // Handle copy from csr
586 if (RISCV::VCSRRegClass.contains(Reg: SrcReg) &&
587 RISCV::GPRRegClass.contains(Reg: DstReg)) {
588 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::CSRRS), DestReg: DstReg)
589 .addImm(Val: RISCVSysReg::lookupSysRegByName(Name: TRI->getName(RegNo: SrcReg))->Encoding)
590 .addReg(RegNo: RISCV::X0);
591 return;
592 }
593
594 if (RISCV::FPR16RegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
595 unsigned Opc;
596 if (STI.hasStdExtZfh()) {
597 Opc = RISCV::FSGNJ_H;
598 } else {
599 assert(STI.hasStdExtF() &&
600 (STI.hasStdExtZfhmin() || STI.hasStdExtZfbfmin()) &&
601 "Unexpected extensions");
602 // Zfhmin/Zfbfmin doesn't have FSGNJ_H, replace FSGNJ_H with FSGNJ_S.
603 DstReg = TRI->getMatchingSuperReg(Reg: DstReg, SubIdx: RISCV::sub_16,
604 RC: &RISCV::FPR32RegClass);
605 SrcReg = TRI->getMatchingSuperReg(Reg: SrcReg, SubIdx: RISCV::sub_16,
606 RC: &RISCV::FPR32RegClass);
607 Opc = RISCV::FSGNJ_S;
608 }
609 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: Opc), DestReg: DstReg)
610 .addReg(RegNo: SrcReg, Flags: KillFlag)
611 .addReg(RegNo: SrcReg, Flags: KillFlag);
612 return;
613 }
614
615 if (RISCV::FPR32RegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
616 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FSGNJ_S), DestReg: DstReg)
617 .addReg(RegNo: SrcReg, Flags: KillFlag)
618 .addReg(RegNo: SrcReg, Flags: KillFlag);
619 return;
620 }
621
622 if (RISCV::FPR64RegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
623 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FSGNJ_D), DestReg: DstReg)
624 .addReg(RegNo: SrcReg, Flags: KillFlag)
625 .addReg(RegNo: SrcReg, Flags: KillFlag);
626 return;
627 }
628
629 if (RISCV::FPR32RegClass.contains(Reg: DstReg) &&
630 RISCV::GPRRegClass.contains(Reg: SrcReg)) {
631 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FMV_W_X), DestReg: DstReg)
632 .addReg(RegNo: SrcReg, Flags: KillFlag);
633 return;
634 }
635
636 if (RISCV::GPRRegClass.contains(Reg: DstReg) &&
637 RISCV::FPR32RegClass.contains(Reg: SrcReg)) {
638 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FMV_X_W), DestReg: DstReg)
639 .addReg(RegNo: SrcReg, Flags: KillFlag);
640 return;
641 }
642
643 if (RISCV::FPR64RegClass.contains(Reg: DstReg) &&
644 RISCV::GPRRegClass.contains(Reg: SrcReg)) {
645 assert(STI.getXLen() == 64 && "Unexpected GPR size");
646 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FMV_D_X), DestReg: DstReg)
647 .addReg(RegNo: SrcReg, Flags: KillFlag);
648 return;
649 }
650
651 if (RISCV::GPRRegClass.contains(Reg: DstReg) &&
652 RISCV::FPR64RegClass.contains(Reg: SrcReg)) {
653 assert(STI.getXLen() == 64 && "Unexpected GPR size");
654 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FMV_X_D), DestReg: DstReg)
655 .addReg(RegNo: SrcReg, Flags: KillFlag);
656 return;
657 }
658
659 // VR->VR copies.
660 const TargetRegisterClass *RegClass =
661 TRI->getCommonMinimalPhysRegClass(Reg1: SrcReg, Reg2: DstReg);
662 if (RISCVRegisterInfo::isRVVRegClass(RC: RegClass)) {
663 copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RegClass);
664 return;
665 }
666
667 llvm_unreachable("Impossible reg-to-reg copy");
668}
669
670void RISCVInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
671 MachineBasicBlock::iterator I,
672 Register SrcReg, bool IsKill, int FI,
673 const TargetRegisterClass *RC,
674 Register VReg,
675 MachineInstr::MIFlag Flags) const {
676 MachineFunction *MF = MBB.getParent();
677 MachineFrameInfo &MFI = MF->getFrameInfo();
678 Align Alignment = MFI.getObjectAlign(ObjectIdx: FI);
679
680 unsigned Opcode;
681 if (RISCV::GPRRegClass.hasSubClassEq(RC)) {
682 Opcode = RegInfo.getRegSizeInBits(RC: RISCV::GPRRegClass) == 32 ? RISCV::SW
683 : RISCV::SD;
684 } else if (RISCV::GPRF16RegClass.hasSubClassEq(RC)) {
685 Opcode = RISCV::SH_INX;
686 } else if (RISCV::GPRF32RegClass.hasSubClassEq(RC)) {
687 Opcode = RISCV::SW_INX;
688 } else if (RISCV::GPRPairRegClass.hasSubClassEq(RC)) {
689 if (!STI.is64Bit() && STI.hasStdExtZilsd() &&
690 Alignment >= STI.getZilsdAlign()) {
691 Opcode = RISCV::SD_RV32;
692 } else {
693 Opcode = RISCV::PseudoRV32ZdinxSD;
694 }
695 } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) {
696 Opcode = RISCV::FSH;
697 } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) {
698 Opcode = RISCV::FSW;
699 } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) {
700 Opcode = RISCV::FSD;
701 } else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
702 Opcode = RISCV::VS1R_V;
703 } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
704 Opcode = RISCV::VS2R_V;
705 } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
706 Opcode = RISCV::VS4R_V;
707 } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
708 Opcode = RISCV::VS8R_V;
709 } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC))
710 Opcode = RISCV::PseudoVSPILL2_M1;
711 else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC))
712 Opcode = RISCV::PseudoVSPILL2_M2;
713 else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC))
714 Opcode = RISCV::PseudoVSPILL2_M4;
715 else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC))
716 Opcode = RISCV::PseudoVSPILL3_M1;
717 else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC))
718 Opcode = RISCV::PseudoVSPILL3_M2;
719 else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC))
720 Opcode = RISCV::PseudoVSPILL4_M1;
721 else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC))
722 Opcode = RISCV::PseudoVSPILL4_M2;
723 else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC))
724 Opcode = RISCV::PseudoVSPILL5_M1;
725 else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC))
726 Opcode = RISCV::PseudoVSPILL6_M1;
727 else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC))
728 Opcode = RISCV::PseudoVSPILL7_M1;
729 else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC))
730 Opcode = RISCV::PseudoVSPILL8_M1;
731 else
732 llvm_unreachable("Can't store this register to stack slot");
733
734 if (RISCVRegisterInfo::isRVVRegClass(RC)) {
735 MachineMemOperand *MMO = MF->getMachineMemOperand(
736 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), F: MachineMemOperand::MOStore,
737 Size: TypeSize::getScalable(MinimumSize: MFI.getObjectSize(ObjectIdx: FI)), BaseAlignment: Alignment);
738
739 MFI.setStackID(ObjectIdx: FI, ID: TargetStackID::ScalableVector);
740 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode))
741 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: IsKill))
742 .addFrameIndex(Idx: FI)
743 .addMemOperand(MMO)
744 .setMIFlag(Flags);
745 NumVRegSpilled += RegInfo.getRegSizeInBits(RC: *RC) / RISCV::RVVBitsPerBlock;
746 } else {
747 MachineMemOperand *MMO = MF->getMachineMemOperand(
748 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), F: MachineMemOperand::MOStore,
749 Size: MFI.getObjectSize(ObjectIdx: FI), BaseAlignment: Alignment);
750
751 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode))
752 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: IsKill))
753 .addFrameIndex(Idx: FI)
754 .addImm(Val: 0)
755 .addMemOperand(MMO)
756 .setMIFlag(Flags);
757 }
758}
759
760void RISCVInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
761 MachineBasicBlock::iterator I,
762 Register DstReg, int FI,
763 const TargetRegisterClass *RC,
764 Register VReg, unsigned SubReg,
765 MachineInstr::MIFlag Flags) const {
766 MachineFunction *MF = MBB.getParent();
767 MachineFrameInfo &MFI = MF->getFrameInfo();
768 Align Alignment = MFI.getObjectAlign(ObjectIdx: FI);
769 DebugLoc DL =
770 Flags & MachineInstr::FrameDestroy ? MBB.findDebugLoc(MBBI: I) : DebugLoc();
771
772 unsigned Opcode;
773 if (RISCV::GPRRegClass.hasSubClassEq(RC)) {
774 Opcode = RegInfo.getRegSizeInBits(RC: RISCV::GPRRegClass) == 32 ? RISCV::LW
775 : RISCV::LD;
776 } else if (RISCV::GPRF16RegClass.hasSubClassEq(RC)) {
777 Opcode = RISCV::LH_INX;
778 } else if (RISCV::GPRF32RegClass.hasSubClassEq(RC)) {
779 Opcode = RISCV::LW_INX;
780 } else if (RISCV::GPRPairRegClass.hasSubClassEq(RC)) {
781 if (!STI.is64Bit() && STI.hasStdExtZilsd() &&
782 Alignment >= STI.getZilsdAlign()) {
783 Opcode = RISCV::LD_RV32;
784 } else {
785 Opcode = RISCV::PseudoRV32ZdinxLD;
786 }
787 } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) {
788 Opcode = RISCV::FLH;
789 } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) {
790 Opcode = RISCV::FLW;
791 } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) {
792 Opcode = RISCV::FLD;
793 } else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
794 Opcode = RISCV::VL1RE8_V;
795 } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
796 Opcode = RISCV::VL2RE8_V;
797 } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
798 Opcode = RISCV::VL4RE8_V;
799 } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
800 Opcode = RISCV::VL8RE8_V;
801 } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC))
802 Opcode = RISCV::PseudoVRELOAD2_M1;
803 else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC))
804 Opcode = RISCV::PseudoVRELOAD2_M2;
805 else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC))
806 Opcode = RISCV::PseudoVRELOAD2_M4;
807 else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC))
808 Opcode = RISCV::PseudoVRELOAD3_M1;
809 else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC))
810 Opcode = RISCV::PseudoVRELOAD3_M2;
811 else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC))
812 Opcode = RISCV::PseudoVRELOAD4_M1;
813 else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC))
814 Opcode = RISCV::PseudoVRELOAD4_M2;
815 else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC))
816 Opcode = RISCV::PseudoVRELOAD5_M1;
817 else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC))
818 Opcode = RISCV::PseudoVRELOAD6_M1;
819 else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC))
820 Opcode = RISCV::PseudoVRELOAD7_M1;
821 else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC))
822 Opcode = RISCV::PseudoVRELOAD8_M1;
823 else
824 llvm_unreachable("Can't load this register from stack slot");
825
826 if (RISCVRegisterInfo::isRVVRegClass(RC)) {
827 MachineMemOperand *MMO = MF->getMachineMemOperand(
828 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), F: MachineMemOperand::MOLoad,
829 Size: TypeSize::getScalable(MinimumSize: MFI.getObjectSize(ObjectIdx: FI)), BaseAlignment: Alignment);
830
831 MFI.setStackID(ObjectIdx: FI, ID: TargetStackID::ScalableVector);
832 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode), DestReg: DstReg)
833 .addFrameIndex(Idx: FI)
834 .addMemOperand(MMO)
835 .setMIFlag(Flags);
836 NumVRegReloaded += RegInfo.getRegSizeInBits(RC: *RC) / RISCV::RVVBitsPerBlock;
837 } else {
838 MachineMemOperand *MMO = MF->getMachineMemOperand(
839 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), F: MachineMemOperand::MOLoad,
840 Size: MFI.getObjectSize(ObjectIdx: FI), BaseAlignment: Alignment);
841
842 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode), DestReg: DstReg)
843 .addFrameIndex(Idx: FI)
844 .addImm(Val: 0)
845 .addMemOperand(MMO)
846 .setMIFlag(Flags);
847 }
848}
849std::optional<unsigned> getFoldedOpcode(MachineFunction &MF, MachineInstr &MI,
850 ArrayRef<unsigned> Ops,
851 const RISCVSubtarget &ST) {
852
853 // The below optimizations narrow the load so they are only valid for little
854 // endian.
855 // TODO: Support big endian by adding an offset into the frame object?
856 if (MF.getDataLayout().isBigEndian())
857 return std::nullopt;
858
859 // Fold load from stack followed by sext.b/sext.h/sext.w/zext.b/zext.h/zext.w.
860 if (Ops.size() != 1 || Ops[0] != 1)
861 return std::nullopt;
862
863 switch (MI.getOpcode()) {
864 default:
865 if (RISCVInstrInfo::isSEXT_W(MI))
866 return RISCV::LW;
867 if (RISCVInstrInfo::isZEXT_W(MI))
868 return RISCV::LWU;
869 if (RISCVInstrInfo::isZEXT_B(MI))
870 return RISCV::LBU;
871 break;
872 case RISCV::SEXT_H:
873 return RISCV::LH;
874 case RISCV::SEXT_B:
875 return RISCV::LB;
876 case RISCV::ZEXT_H_RV32:
877 case RISCV::ZEXT_H_RV64:
878 return RISCV::LHU;
879 }
880
881 switch (RISCV::getRVVMCOpcode(RVVPseudoOpcode: MI.getOpcode())) {
882 default:
883 return std::nullopt;
884 case RISCV::VMV_X_S: {
885 unsigned Log2SEW =
886 MI.getOperand(i: RISCVII::getSEWOpNum(Desc: MI.getDesc())).getImm();
887 if (ST.getXLen() < (1U << Log2SEW))
888 return std::nullopt;
889 switch (Log2SEW) {
890 case 3:
891 return RISCV::LB;
892 case 4:
893 return RISCV::LH;
894 case 5:
895 return RISCV::LW;
896 case 6:
897 return RISCV::LD;
898 default:
899 llvm_unreachable("Unexpected SEW");
900 }
901 }
902 case RISCV::VFMV_F_S: {
903 unsigned Log2SEW =
904 MI.getOperand(i: RISCVII::getSEWOpNum(Desc: MI.getDesc())).getImm();
905 switch (Log2SEW) {
906 case 4:
907 return RISCV::FLH;
908 case 5:
909 return RISCV::FLW;
910 case 6:
911 return RISCV::FLD;
912 default:
913 llvm_unreachable("Unexpected SEW");
914 }
915 }
916 }
917}
918
919// This is the version used during InlineSpiller::spillAroundUses
920MachineInstr *
921RISCVInstrInfo::foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
922 ArrayRef<unsigned> Ops, int FrameIndex,
923 MachineInstr *&CopyMI, LiveIntervals *LIS,
924 VirtRegMap *VRM) const {
925 MachineBasicBlock::iterator InsertPt = MI;
926 std::optional<unsigned> LoadOpc = getFoldedOpcode(MF, MI, Ops, ST: STI);
927 if (!LoadOpc)
928 return nullptr;
929 Register DstReg = MI.getOperand(i: 0).getReg();
930 return BuildMI(BB&: *MI.getParent(), I: InsertPt, MIMD: MI.getDebugLoc(), MCID: get(Opcode: *LoadOpc),
931 DestReg: DstReg)
932 .addFrameIndex(Idx: FrameIndex)
933 .addImm(Val: 0);
934}
935
936static unsigned getLoadPredicatedOpcode(unsigned Opcode) {
937 switch (Opcode) {
938 case RISCV::LB:
939 return RISCV::PseudoCCLB;
940 case RISCV::LBU:
941 return RISCV::PseudoCCLBU;
942 case RISCV::LH:
943 return RISCV::PseudoCCLH;
944 case RISCV::LHU:
945 return RISCV::PseudoCCLHU;
946 case RISCV::LW:
947 return RISCV::PseudoCCLW;
948 case RISCV::LWU:
949 return RISCV::PseudoCCLWU;
950 case RISCV::LD:
951 return RISCV::PseudoCCLD;
952 case RISCV::QC_E_LB:
953 return RISCV::PseudoCCQC_E_LB;
954 case RISCV::QC_E_LBU:
955 return RISCV::PseudoCCQC_E_LBU;
956 case RISCV::QC_E_LH:
957 return RISCV::PseudoCCQC_E_LH;
958 case RISCV::QC_E_LHU:
959 return RISCV::PseudoCCQC_E_LHU;
960 case RISCV::QC_E_LW:
961 return RISCV::PseudoCCQC_E_LW;
962 default:
963 return 0;
964 }
965}
966
967MachineInstr *RISCVInstrInfo::foldMemoryOperandImpl(
968 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
969 MachineInstr &LoadMI, MachineInstr *&CopyMI, LiveIntervals *LIS,
970 VirtRegMap *VRM) const {
971 MachineBasicBlock::iterator InsertPt = MI;
972 // For now, only handle RISCV::PseudoCCMOVGPR.
973 if (MI.getOpcode() != RISCV::PseudoCCMOVGPR)
974 return nullptr;
975
976 unsigned PredOpc = getLoadPredicatedOpcode(Opcode: LoadMI.getOpcode());
977
978 if (!STI.hasShortForwardBranchILoad() || !PredOpc)
979 return nullptr;
980
981 MachineRegisterInfo &MRI = MF.getRegInfo();
982 if (Ops.size() != 1 || (Ops[0] != 1 && Ops[0] != 2))
983 return nullptr;
984
985 bool Invert = Ops[0] == 2;
986 const MachineOperand &FalseReg = MI.getOperand(i: !Invert ? 2 : 1);
987 Register DestReg = MI.getOperand(i: 0).getReg();
988 const TargetRegisterClass *PreviousClass = MRI.getRegClass(Reg: FalseReg.getReg());
989 if (!MRI.constrainRegClass(Reg: DestReg, RC: PreviousClass))
990 return nullptr;
991
992 // Create a new predicated version of DefMI.
993 MachineInstrBuilder NewMI = BuildMI(BB&: *MI.getParent(), I: InsertPt,
994 MIMD: MI.getDebugLoc(), MCID: get(Opcode: PredOpc), DestReg);
995
996 // Copy the false register.
997 NewMI.add(MO: FalseReg);
998
999 // Copy all the DefMI operands.
1000 const MCInstrDesc &DefDesc = LoadMI.getDesc();
1001 for (unsigned i = 1, e = DefDesc.getNumOperands(); i != e; ++i)
1002 NewMI.add(MO: LoadMI.getOperand(i));
1003
1004 // Add branch opcode, inverting if necessary.
1005 unsigned BCC = MI.getOperand(i: MI.getNumExplicitOperands() - 3).getImm();
1006 if (!Invert)
1007 BCC = RISCVCC::getInverseBranchOpcode(BCC);
1008 NewMI.addImm(Val: BCC);
1009
1010 // Copy condition portion
1011 NewMI.add(MOs: {MI.getOperand(i: MI.getNumExplicitOperands() - 2),
1012 MI.getOperand(i: MI.getNumExplicitOperands() - 1)});
1013 NewMI.cloneMemRefs(OtherMI: LoadMI);
1014 return NewMI;
1015}
1016
1017void RISCVInstrInfo::movImm(MachineBasicBlock &MBB,
1018 MachineBasicBlock::iterator MBBI,
1019 const DebugLoc &DL, Register DstReg, uint64_t Val,
1020 MachineInstr::MIFlag Flag, bool DstRenamable,
1021 bool DstIsDead) const {
1022 Register SrcReg = RISCV::X0;
1023
1024 // For RV32, allow a sign or unsigned 32 bit value.
1025 if (!STI.is64Bit() && !isInt<32>(x: Val)) {
1026 // If have a uimm32 it will still fit in a register so we can allow it.
1027 if (!isUInt<32>(x: Val))
1028 report_fatal_error(reason: "Should only materialize 32-bit constants for RV32");
1029
1030 // Sign extend for generateInstSeq.
1031 Val = SignExtend64<32>(x: Val);
1032 }
1033
1034 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val, STI);
1035 assert(!Seq.empty());
1036
1037 bool SrcRenamable = false;
1038 unsigned Num = 0;
1039
1040 for (const RISCVMatInt::Inst &Inst : Seq) {
1041 bool LastItem = ++Num == Seq.size();
1042 RegState DstRegState = getDeadRegState(B: DstIsDead && LastItem) |
1043 getRenamableRegState(B: DstRenamable);
1044 RegState SrcRegState = getKillRegState(B: SrcReg != RISCV::X0) |
1045 getRenamableRegState(B: SrcRenamable);
1046 switch (Inst.getOpndKind()) {
1047 case RISCVMatInt::Imm:
1048 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: Inst.getOpcode()))
1049 .addReg(RegNo: DstReg, Flags: RegState::Define | DstRegState)
1050 .addImm(Val: Inst.getImm())
1051 .setMIFlag(Flag);
1052 break;
1053 case RISCVMatInt::RegX0:
1054 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: Inst.getOpcode()))
1055 .addReg(RegNo: DstReg, Flags: RegState::Define | DstRegState)
1056 .addReg(RegNo: SrcReg, Flags: SrcRegState)
1057 .addReg(RegNo: RISCV::X0)
1058 .setMIFlag(Flag);
1059 break;
1060 case RISCVMatInt::RegReg:
1061 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: Inst.getOpcode()))
1062 .addReg(RegNo: DstReg, Flags: RegState::Define | DstRegState)
1063 .addReg(RegNo: SrcReg, Flags: SrcRegState)
1064 .addReg(RegNo: SrcReg, Flags: SrcRegState)
1065 .setMIFlag(Flag);
1066 break;
1067 case RISCVMatInt::RegImm:
1068 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: Inst.getOpcode()))
1069 .addReg(RegNo: DstReg, Flags: RegState::Define | DstRegState)
1070 .addReg(RegNo: SrcReg, Flags: SrcRegState)
1071 .addImm(Val: Inst.getImm())
1072 .setMIFlag(Flag);
1073 break;
1074 }
1075
1076 // Only the first instruction has X0 as its source.
1077 SrcReg = DstReg;
1078 SrcRenamable = DstRenamable;
1079 }
1080}
1081
1082RISCVCC::CondCode RISCVInstrInfo::getCondFromBranchOpc(unsigned Opc) {
1083 switch (Opc) {
1084 default:
1085 return RISCVCC::COND_INVALID;
1086 case RISCV::BEQ:
1087 case RISCV::BEQI:
1088 case RISCV::CV_BEQIMM:
1089 case RISCV::QC_BEQI:
1090 case RISCV::QC_E_BEQI:
1091 case RISCV::NDS_BBC:
1092 case RISCV::NDS_BEQC:
1093 return RISCVCC::COND_EQ;
1094 case RISCV::BNE:
1095 case RISCV::BNEI:
1096 case RISCV::QC_BNEI:
1097 case RISCV::QC_E_BNEI:
1098 case RISCV::CV_BNEIMM:
1099 case RISCV::NDS_BBS:
1100 case RISCV::NDS_BNEC:
1101 return RISCVCC::COND_NE;
1102 case RISCV::BLT:
1103 case RISCV::QC_BLTI:
1104 case RISCV::QC_E_BLTI:
1105 return RISCVCC::COND_LT;
1106 case RISCV::BGE:
1107 case RISCV::QC_BGEI:
1108 case RISCV::QC_E_BGEI:
1109 return RISCVCC::COND_GE;
1110 case RISCV::BLTU:
1111 case RISCV::QC_BLTUI:
1112 case RISCV::QC_E_BLTUI:
1113 return RISCVCC::COND_LTU;
1114 case RISCV::BGEU:
1115 case RISCV::QC_BGEUI:
1116 case RISCV::QC_E_BGEUI:
1117 return RISCVCC::COND_GEU;
1118 }
1119}
1120
1121bool RISCVInstrInfo::evaluateCondBranch(RISCVCC::CondCode CC, int64_t C0,
1122 int64_t C1) {
1123 switch (CC) {
1124 default:
1125 llvm_unreachable("Unexpected CC");
1126 case RISCVCC::COND_EQ:
1127 return C0 == C1;
1128 case RISCVCC::COND_NE:
1129 return C0 != C1;
1130 case RISCVCC::COND_LT:
1131 return C0 < C1;
1132 case RISCVCC::COND_GE:
1133 return C0 >= C1;
1134 case RISCVCC::COND_LTU:
1135 return (uint64_t)C0 < (uint64_t)C1;
1136 case RISCVCC::COND_GEU:
1137 return (uint64_t)C0 >= (uint64_t)C1;
1138 }
1139}
1140
1141// The contents of values added to Cond are not examined outside of
1142// RISCVInstrInfo, giving us flexibility in what to push to it. For RISCV, we
1143// push BranchOpcode, Reg1, Reg2.
1144static void parseCondBranch(MachineInstr &LastInst, MachineBasicBlock *&Target,
1145 SmallVectorImpl<MachineOperand> &Cond) {
1146 // Block ends with fall-through condbranch.
1147 assert(LastInst.getDesc().isConditionalBranch() &&
1148 "Unknown conditional branch");
1149 Target = LastInst.getOperand(i: 2).getMBB();
1150 Cond.push_back(Elt: MachineOperand::CreateImm(Val: LastInst.getOpcode()));
1151 Cond.push_back(Elt: LastInst.getOperand(i: 0));
1152 Cond.push_back(Elt: LastInst.getOperand(i: 1));
1153}
1154
1155static unsigned getInverseXqcicmOpcode(unsigned Opcode) {
1156 switch (Opcode) {
1157 default:
1158 llvm_unreachable("Unexpected Opcode");
1159 case RISCV::QC_MVEQ:
1160 return RISCV::QC_MVNE;
1161 case RISCV::QC_MVNE:
1162 return RISCV::QC_MVEQ;
1163 case RISCV::QC_MVLT:
1164 return RISCV::QC_MVGE;
1165 case RISCV::QC_MVGE:
1166 return RISCV::QC_MVLT;
1167 case RISCV::QC_MVLTU:
1168 return RISCV::QC_MVGEU;
1169 case RISCV::QC_MVGEU:
1170 return RISCV::QC_MVLTU;
1171 case RISCV::QC_MVEQI:
1172 return RISCV::QC_MVNEI;
1173 case RISCV::QC_MVNEI:
1174 return RISCV::QC_MVEQI;
1175 case RISCV::QC_MVLTI:
1176 return RISCV::QC_MVGEI;
1177 case RISCV::QC_MVGEI:
1178 return RISCV::QC_MVLTI;
1179 case RISCV::QC_MVLTUI:
1180 return RISCV::QC_MVGEUI;
1181 case RISCV::QC_MVGEUI:
1182 return RISCV::QC_MVLTUI;
1183 }
1184}
1185
1186unsigned RISCVCC::getBrCond(RISCVCC::CondCode CC, unsigned SelectOpc) {
1187 switch (SelectOpc) {
1188 default:
1189 switch (CC) {
1190 default:
1191 llvm_unreachable("Unexpected condition code!");
1192 case RISCVCC::COND_EQ:
1193 return RISCV::BEQ;
1194 case RISCVCC::COND_NE:
1195 return RISCV::BNE;
1196 case RISCVCC::COND_LT:
1197 return RISCV::BLT;
1198 case RISCVCC::COND_GE:
1199 return RISCV::BGE;
1200 case RISCVCC::COND_LTU:
1201 return RISCV::BLTU;
1202 case RISCVCC::COND_GEU:
1203 return RISCV::BGEU;
1204 }
1205 break;
1206 case RISCV::Select_GPR_Using_CC_Imm5_Zibi:
1207 switch (CC) {
1208 default:
1209 llvm_unreachable("Unexpected condition code!");
1210 case RISCVCC::COND_EQ:
1211 return RISCV::BEQI;
1212 case RISCVCC::COND_NE:
1213 return RISCV::BNEI;
1214 }
1215 break;
1216 case RISCV::Select_GPR_Using_CC_SImm5_CV:
1217 switch (CC) {
1218 default:
1219 llvm_unreachable("Unexpected condition code!");
1220 case RISCVCC::COND_EQ:
1221 return RISCV::CV_BEQIMM;
1222 case RISCVCC::COND_NE:
1223 return RISCV::CV_BNEIMM;
1224 }
1225 break;
1226 case RISCV::Select_GPRNoX0_Using_CC_SImm5NonZero_QC:
1227 switch (CC) {
1228 default:
1229 llvm_unreachable("Unexpected condition code!");
1230 case RISCVCC::COND_EQ:
1231 return RISCV::QC_BEQI;
1232 case RISCVCC::COND_NE:
1233 return RISCV::QC_BNEI;
1234 case RISCVCC::COND_LT:
1235 return RISCV::QC_BLTI;
1236 case RISCVCC::COND_GE:
1237 return RISCV::QC_BGEI;
1238 }
1239 break;
1240 case RISCV::Select_GPRNoX0_Using_CC_UImm5NonZero_QC:
1241 switch (CC) {
1242 default:
1243 llvm_unreachable("Unexpected condition code!");
1244 case RISCVCC::COND_LTU:
1245 return RISCV::QC_BLTUI;
1246 case RISCVCC::COND_GEU:
1247 return RISCV::QC_BGEUI;
1248 }
1249 break;
1250 case RISCV::Select_GPRNoX0_Using_CC_SImm16NonZero_QC:
1251 switch (CC) {
1252 default:
1253 llvm_unreachable("Unexpected condition code!");
1254 case RISCVCC::COND_EQ:
1255 return RISCV::QC_E_BEQI;
1256 case RISCVCC::COND_NE:
1257 return RISCV::QC_E_BNEI;
1258 case RISCVCC::COND_LT:
1259 return RISCV::QC_E_BLTI;
1260 case RISCVCC::COND_GE:
1261 return RISCV::QC_E_BGEI;
1262 }
1263 break;
1264 case RISCV::Select_GPRNoX0_Using_CC_UImm16NonZero_QC:
1265 switch (CC) {
1266 default:
1267 llvm_unreachable("Unexpected condition code!");
1268 case RISCVCC::COND_LTU:
1269 return RISCV::QC_E_BLTUI;
1270 case RISCVCC::COND_GEU:
1271 return RISCV::QC_E_BGEUI;
1272 }
1273 break;
1274 case RISCV::Select_GPR_Using_CC_UImmLog2XLen_NDS:
1275 switch (CC) {
1276 default:
1277 llvm_unreachable("Unexpected condition code!");
1278 case RISCVCC::COND_EQ:
1279 return RISCV::NDS_BBC;
1280 case RISCVCC::COND_NE:
1281 return RISCV::NDS_BBS;
1282 }
1283 break;
1284 case RISCV::Select_GPR_Using_CC_UImm7_NDS:
1285 switch (CC) {
1286 default:
1287 llvm_unreachable("Unexpected condition code!");
1288 case RISCVCC::COND_EQ:
1289 return RISCV::NDS_BEQC;
1290 case RISCVCC::COND_NE:
1291 return RISCV::NDS_BNEC;
1292 }
1293 break;
1294 }
1295}
1296
1297RISCVCC::CondCode RISCVCC::getInverseBranchCondition(RISCVCC::CondCode CC) {
1298 switch (CC) {
1299 default:
1300 llvm_unreachable("Unrecognized conditional branch");
1301 case RISCVCC::COND_EQ:
1302 return RISCVCC::COND_NE;
1303 case RISCVCC::COND_NE:
1304 return RISCVCC::COND_EQ;
1305 case RISCVCC::COND_LT:
1306 return RISCVCC::COND_GE;
1307 case RISCVCC::COND_GE:
1308 return RISCVCC::COND_LT;
1309 case RISCVCC::COND_LTU:
1310 return RISCVCC::COND_GEU;
1311 case RISCVCC::COND_GEU:
1312 return RISCVCC::COND_LTU;
1313 }
1314}
1315
1316// Return inverse branch
1317unsigned RISCVCC::getInverseBranchOpcode(unsigned BCC) {
1318 switch (BCC) {
1319 default:
1320 llvm_unreachable("Unexpected branch opcode!");
1321 case RISCV::BEQ:
1322 return RISCV::BNE;
1323 case RISCV::BEQI:
1324 return RISCV::BNEI;
1325 case RISCV::BNE:
1326 return RISCV::BEQ;
1327 case RISCV::BNEI:
1328 return RISCV::BEQI;
1329 case RISCV::BLT:
1330 return RISCV::BGE;
1331 case RISCV::BGE:
1332 return RISCV::BLT;
1333 case RISCV::BLTU:
1334 return RISCV::BGEU;
1335 case RISCV::BGEU:
1336 return RISCV::BLTU;
1337 case RISCV::CV_BEQIMM:
1338 return RISCV::CV_BNEIMM;
1339 case RISCV::CV_BNEIMM:
1340 return RISCV::CV_BEQIMM;
1341 case RISCV::QC_BEQI:
1342 return RISCV::QC_BNEI;
1343 case RISCV::QC_BNEI:
1344 return RISCV::QC_BEQI;
1345 case RISCV::QC_BLTI:
1346 return RISCV::QC_BGEI;
1347 case RISCV::QC_BGEI:
1348 return RISCV::QC_BLTI;
1349 case RISCV::QC_BLTUI:
1350 return RISCV::QC_BGEUI;
1351 case RISCV::QC_BGEUI:
1352 return RISCV::QC_BLTUI;
1353 case RISCV::QC_E_BEQI:
1354 return RISCV::QC_E_BNEI;
1355 case RISCV::QC_E_BNEI:
1356 return RISCV::QC_E_BEQI;
1357 case RISCV::QC_E_BLTI:
1358 return RISCV::QC_E_BGEI;
1359 case RISCV::QC_E_BGEI:
1360 return RISCV::QC_E_BLTI;
1361 case RISCV::QC_E_BLTUI:
1362 return RISCV::QC_E_BGEUI;
1363 case RISCV::QC_E_BGEUI:
1364 return RISCV::QC_E_BLTUI;
1365 case RISCV::NDS_BBC:
1366 return RISCV::NDS_BBS;
1367 case RISCV::NDS_BBS:
1368 return RISCV::NDS_BBC;
1369 case RISCV::NDS_BEQC:
1370 return RISCV::NDS_BNEC;
1371 case RISCV::NDS_BNEC:
1372 return RISCV::NDS_BEQC;
1373 }
1374}
1375
1376bool RISCVInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
1377 MachineBasicBlock *&TBB,
1378 MachineBasicBlock *&FBB,
1379 SmallVectorImpl<MachineOperand> &Cond,
1380 bool AllowModify) const {
1381 TBB = FBB = nullptr;
1382 Cond.clear();
1383
1384 // If the block has no terminators, it just falls into the block after it.
1385 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
1386 if (I == MBB.end() || !isUnpredicatedTerminator(MI: *I))
1387 return false;
1388
1389 // Count the number of terminators and find the first unconditional or
1390 // indirect branch.
1391 MachineBasicBlock::iterator FirstUncondOrIndirectBr = MBB.end();
1392 int NumTerminators = 0;
1393 for (auto J = I.getReverse(); J != MBB.rend() && isUnpredicatedTerminator(MI: *J);
1394 J++) {
1395 NumTerminators++;
1396 if (J->getDesc().isUnconditionalBranch() ||
1397 J->getDesc().isIndirectBranch()) {
1398 FirstUncondOrIndirectBr = J.getReverse();
1399 }
1400 }
1401
1402 // If AllowModify is true, we can erase any terminators after
1403 // FirstUncondOrIndirectBR.
1404 if (AllowModify && FirstUncondOrIndirectBr != MBB.end()) {
1405 while (std::next(x: FirstUncondOrIndirectBr) != MBB.end()) {
1406 std::next(x: FirstUncondOrIndirectBr)->eraseFromParent();
1407 NumTerminators--;
1408 }
1409 I = FirstUncondOrIndirectBr;
1410 }
1411
1412 // We can't handle blocks that end in an indirect branch.
1413 if (I->getDesc().isIndirectBranch())
1414 return true;
1415
1416 // We can't handle Generic branch opcodes from Global ISel.
1417 if (I->isPreISelOpcode())
1418 return true;
1419
1420 // We can't handle blocks with more than 2 terminators.
1421 if (NumTerminators > 2)
1422 return true;
1423
1424 // Handle a single unconditional branch.
1425 if (NumTerminators == 1 && I->getDesc().isUnconditionalBranch()) {
1426 TBB = getBranchDestBlock(MI: *I);
1427 return false;
1428 }
1429
1430 // Handle a single conditional branch.
1431 if (NumTerminators == 1 && I->getDesc().isConditionalBranch()) {
1432 parseCondBranch(LastInst&: *I, Target&: TBB, Cond);
1433 return false;
1434 }
1435
1436 // Handle a conditional branch followed by an unconditional branch.
1437 if (NumTerminators == 2 && std::prev(x: I)->getDesc().isConditionalBranch() &&
1438 I->getDesc().isUnconditionalBranch()) {
1439 parseCondBranch(LastInst&: *std::prev(x: I), Target&: TBB, Cond);
1440 FBB = getBranchDestBlock(MI: *I);
1441 return false;
1442 }
1443
1444 // Otherwise, we can't handle this.
1445 return true;
1446}
1447
1448unsigned RISCVInstrInfo::removeBranch(MachineBasicBlock &MBB,
1449 int *BytesRemoved) const {
1450 if (BytesRemoved)
1451 *BytesRemoved = 0;
1452 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
1453 if (I == MBB.end())
1454 return 0;
1455
1456 if (!I->getDesc().isUnconditionalBranch() &&
1457 !I->getDesc().isConditionalBranch())
1458 return 0;
1459
1460 // Remove the branch.
1461 if (BytesRemoved)
1462 *BytesRemoved += getInstSizeInBytes(MI: *I);
1463 I->eraseFromParent();
1464
1465 I = MBB.end();
1466
1467 if (I == MBB.begin())
1468 return 1;
1469 --I;
1470 if (!I->getDesc().isConditionalBranch())
1471 return 1;
1472
1473 // Remove the branch.
1474 if (BytesRemoved)
1475 *BytesRemoved += getInstSizeInBytes(MI: *I);
1476 I->eraseFromParent();
1477 return 2;
1478}
1479
1480// Inserts a branch into the end of the specific MachineBasicBlock, returning
1481// the number of instructions inserted.
1482unsigned RISCVInstrInfo::insertBranch(
1483 MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
1484 ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const {
1485 if (BytesAdded)
1486 *BytesAdded = 0;
1487
1488 // Shouldn't be a fall through.
1489 assert(TBB && "insertBranch must not be told to insert a fallthrough");
1490 assert((Cond.size() == 3 || Cond.size() == 0) &&
1491 "RISC-V branch conditions have two components!");
1492
1493 // Unconditional branch.
1494 if (Cond.empty()) {
1495 MachineInstr &MI = *BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: RISCV::PseudoBR)).addMBB(MBB: TBB);
1496 if (BytesAdded)
1497 *BytesAdded += getInstSizeInBytes(MI);
1498 return 1;
1499 }
1500
1501 // Either a one or two-way conditional branch.
1502 MachineInstr &CondMI = *BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: Cond[0].getImm()))
1503 .add(MO: Cond[1])
1504 .add(MO: Cond[2])
1505 .addMBB(MBB: TBB);
1506 if (BytesAdded)
1507 *BytesAdded += getInstSizeInBytes(MI: CondMI);
1508
1509 // One-way conditional branch.
1510 if (!FBB)
1511 return 1;
1512
1513 // Two-way conditional branch.
1514 MachineInstr &MI = *BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: RISCV::PseudoBR)).addMBB(MBB: FBB);
1515 if (BytesAdded)
1516 *BytesAdded += getInstSizeInBytes(MI);
1517 return 2;
1518}
1519
1520void RISCVInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
1521 MachineBasicBlock &DestBB,
1522 MachineBasicBlock &RestoreBB,
1523 const DebugLoc &DL, int64_t BrOffset,
1524 RegScavenger *RS) const {
1525 assert(RS && "RegScavenger required for long branching");
1526 assert(MBB.empty() &&
1527 "new block should be inserted for expanding unconditional branch");
1528 assert(MBB.pred_size() == 1);
1529 assert(RestoreBB.empty() &&
1530 "restore block should be inserted for restoring clobbered registers");
1531
1532 MachineFunction *MF = MBB.getParent();
1533 MachineRegisterInfo &MRI = MF->getRegInfo();
1534 RISCVMachineFunctionInfo *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
1535 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
1536
1537 if (!isInt<32>(x: BrOffset))
1538 report_fatal_error(
1539 reason: "Branch offsets outside of the signed 32-bit range not supported");
1540
1541 // FIXME: A virtual register must be used initially, as the register
1542 // scavenger won't work with empty blocks (SIInstrInfo::insertIndirectBranch
1543 // uses the same workaround).
1544 Register ScratchReg = MRI.createVirtualRegister(RegClass: &RISCV::GPRJALRRegClass);
1545 auto II = MBB.end();
1546 // We may also update the jump target to RestoreBB later.
1547 MachineInstr &MI = *BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::PseudoJump))
1548 .addReg(RegNo: ScratchReg, Flags: RegState::Define | RegState::Dead)
1549 .addMBB(MBB: &DestBB, TargetFlags: RISCVII::MO_CALL);
1550
1551 RS->enterBasicBlockEnd(MBB);
1552 // When cf-protection-branch is enabled, we must use t2 (x7) for software
1553 // guarded branches to hold the landing pad label.
1554 bool HasCFBranch =
1555 MF->getInfo<RISCVMachineFunctionInfo>()->hasCFProtectionBranch();
1556 const TargetRegisterClass *RC = &RISCV::GPRRegClass;
1557 if (HasCFBranch)
1558 RC = &RISCV::GPRX7RegClass;
1559 Register TmpGPR =
1560 RS->scavengeRegisterBackwards(RC: *RC, To: MI.getIterator(),
1561 /*RestoreAfter=*/false, /*SpAdj=*/SPAdj: 0,
1562 /*AllowSpill=*/false);
1563 if (TmpGPR.isValid())
1564 RS->setRegUsed(Reg: TmpGPR);
1565 else {
1566 // The case when there is no scavenged register needs special handling.
1567
1568 // Pick s11(or s1 for rve) because it doesn't make a difference.
1569 TmpGPR = STI.hasStdExtE() ? RISCV::X9 : RISCV::X27;
1570 // Force t2 if cf-protection-branch is enabled
1571 if (HasCFBranch)
1572 TmpGPR = RISCV::X7;
1573
1574 int FrameIndex = RVFI->getBranchRelaxationScratchFrameIndex();
1575 if (FrameIndex == -1)
1576 report_fatal_error(reason: "underestimated function size");
1577
1578 storeRegToStackSlot(MBB, I: MI, SrcReg: TmpGPR, /*IsKill=*/true, FI: FrameIndex,
1579 RC: &RISCV::GPRRegClass, VReg: Register());
1580 TRI->eliminateFrameIndex(MI: std::prev(x: MI.getIterator()),
1581 /*SpAdj=*/SPAdj: 0, /*FIOperandNum=*/1);
1582
1583 MI.getOperand(i: 1).setMBB(&RestoreBB);
1584
1585 loadRegFromStackSlot(MBB&: RestoreBB, I: RestoreBB.end(), DstReg: TmpGPR, FI: FrameIndex,
1586 RC: &RISCV::GPRRegClass, VReg: Register());
1587 TRI->eliminateFrameIndex(MI: RestoreBB.back(),
1588 /*SpAdj=*/SPAdj: 0, /*FIOperandNum=*/1);
1589 }
1590
1591 MRI.replaceRegWith(FromReg: ScratchReg, ToReg: TmpGPR);
1592 MRI.clearVirtRegs();
1593}
1594
1595bool RISCVInstrInfo::reverseBranchCondition(
1596 SmallVectorImpl<MachineOperand> &Cond) const {
1597 assert((Cond.size() == 3) && "Invalid branch condition!");
1598
1599 Cond[0].setImm(RISCVCC::getInverseBranchOpcode(BCC: Cond[0].getImm()));
1600
1601 return false;
1602}
1603
1604// Return true if the instruction is a load immediate instruction (i.e.
1605// (ADDI x0, imm) or (BSETI x0, imm)).
1606static bool isLoadImm(const MachineInstr *MI, int64_t &Imm) {
1607 if (MI->getOpcode() == RISCV::ADDI && MI->getOperand(i: 1).isReg() &&
1608 MI->getOperand(i: 1).getReg() == RISCV::X0) {
1609 Imm = MI->getOperand(i: 2).getImm();
1610 return true;
1611 }
1612 // BSETI can be used to create power of 2 constants. Only 2048 is currently
1613 // interesting because it is 1 more than the maximum ADDI constant.
1614 if (MI->getOpcode() == RISCV::BSETI && MI->getOperand(i: 1).isReg() &&
1615 MI->getOperand(i: 1).getReg() == RISCV::X0 &&
1616 MI->getOperand(i: 2).getImm() == 11) {
1617 Imm = 2048;
1618 return true;
1619 }
1620 return false;
1621}
1622
1623bool RISCVInstrInfo::isFromLoadImm(const MachineRegisterInfo &MRI,
1624 const MachineOperand &Op, int64_t &Imm) {
1625 // Either a load from immediate instruction or X0.
1626 if (!Op.isReg())
1627 return false;
1628
1629 Register Reg = Op.getReg();
1630 if (Reg == RISCV::X0) {
1631 Imm = 0;
1632 return true;
1633 }
1634
1635 if (!Reg.isVirtual())
1636 return false;
1637
1638 const MachineInstr *DefMI = MRI.getVRegDef(Reg);
1639 return DefMI && isLoadImm(MI: DefMI, Imm);
1640}
1641
1642bool RISCVInstrInfo::optimizeCondBranch(MachineInstr &MI) const {
1643 bool IsSigned = false;
1644 bool IsEquality = false;
1645 switch (MI.getOpcode()) {
1646 default:
1647 return false;
1648 case RISCV::BEQ:
1649 case RISCV::BNE:
1650 IsEquality = true;
1651 break;
1652 case RISCV::BGE:
1653 case RISCV::BLT:
1654 IsSigned = true;
1655 break;
1656 case RISCV::BGEU:
1657 case RISCV::BLTU:
1658 break;
1659 }
1660
1661 MachineBasicBlock *MBB = MI.getParent();
1662 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
1663
1664 const MachineOperand &LHS = MI.getOperand(i: 0);
1665 const MachineOperand &RHS = MI.getOperand(i: 1);
1666 MachineBasicBlock *TBB = MI.getOperand(i: 2).getMBB();
1667
1668 RISCVCC::CondCode CC = getCondFromBranchOpc(Opc: MI.getOpcode());
1669 assert(CC != RISCVCC::COND_INVALID);
1670
1671 // Canonicalize conditional branches which can be constant folded into
1672 // beqz or bnez. We can't modify the CFG here.
1673 int64_t C0, C1;
1674 if (isFromLoadImm(MRI, Op: LHS, Imm&: C0) && isFromLoadImm(MRI, Op: RHS, Imm&: C1)) {
1675 unsigned NewOpc = evaluateCondBranch(CC, C0, C1) ? RISCV::BEQ : RISCV::BNE;
1676 // Build the new branch and remove the old one.
1677 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
1678 .addReg(RegNo: RISCV::X0)
1679 .addReg(RegNo: RISCV::X0)
1680 .addMBB(MBB: TBB);
1681 MI.eraseFromParent();
1682 return true;
1683 }
1684
1685 if (IsEquality)
1686 return false;
1687
1688 // For two constants C0 and C1 from
1689 // ```
1690 // li Y, C0
1691 // li Z, C1
1692 // ```
1693 // 1. if C1 = C0 + 1
1694 // we can turn:
1695 // (a) blt Y, X -> bge X, Z
1696 // (b) bge Y, X -> blt X, Z
1697 //
1698 // 2. if C1 = C0 - 1
1699 // we can turn:
1700 // (a) blt X, Y -> bge Z, X
1701 // (b) bge X, Y -> blt Z, X
1702 //
1703 // To make sure this optimization is really beneficial, we only
1704 // optimize for cases where Y had only one use (i.e. only used by the branch).
1705 // Try to find the register for constant Z; return
1706 // invalid register otherwise.
1707 auto searchConst = [&](int64_t C1) -> Register {
1708 MachineBasicBlock::reverse_iterator II(&MI), E = MBB->rend();
1709 auto DefC1 = std::find_if(first: ++II, last: E, pred: [&](const MachineInstr &I) -> bool {
1710 int64_t Imm;
1711 return isLoadImm(MI: &I, Imm) && Imm == C1 &&
1712 I.getOperand(i: 0).getReg().isVirtual();
1713 });
1714 if (DefC1 != E)
1715 return DefC1->getOperand(i: 0).getReg();
1716
1717 return Register();
1718 };
1719
1720 unsigned NewOpc = RISCVCC::getBrCond(CC: getInverseBranchCondition(CC));
1721
1722 // Might be case 1.
1723 // Don't change 0 to 1 since we can use x0.
1724 // For unsigned cases changing -1U to 0 would be incorrect.
1725 // The incorrect case for signed would be INT_MAX, but isFromLoadImm can't
1726 // return that.
1727 if (isFromLoadImm(MRI, Op: LHS, Imm&: C0) && C0 != 0 && LHS.getReg().isVirtual() &&
1728 MRI.hasOneUse(RegNo: LHS.getReg()) && (IsSigned || C0 != -1)) {
1729 assert((isInt<12>(C0) || C0 == 2048) && "Unexpected immediate");
1730 if (Register RegZ = searchConst(C0 + 1)) {
1731 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
1732 .add(MO: RHS)
1733 .addReg(RegNo: RegZ)
1734 .addMBB(MBB: TBB);
1735 // We might extend the live range of Z, clear its kill flag to
1736 // account for this.
1737 MRI.clearKillFlags(Reg: RegZ);
1738 MI.eraseFromParent();
1739 return true;
1740 }
1741 }
1742
1743 // Might be case 2.
1744 // For signed cases we don't want to change 0 since we can use x0.
1745 // For unsigned cases changing 0 to -1U would be incorrect.
1746 // The incorrect case for signed would be INT_MIN, but isFromLoadImm can't
1747 // return that.
1748 if (isFromLoadImm(MRI, Op: RHS, Imm&: C0) && C0 != 0 && RHS.getReg().isVirtual() &&
1749 MRI.hasOneUse(RegNo: RHS.getReg())) {
1750 assert((isInt<12>(C0) || C0 == 2048) && "Unexpected immediate");
1751 if (Register RegZ = searchConst(C0 - 1)) {
1752 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
1753 .addReg(RegNo: RegZ)
1754 .add(MO: LHS)
1755 .addMBB(MBB: TBB);
1756 // We might extend the live range of Z, clear its kill flag to
1757 // account for this.
1758 MRI.clearKillFlags(Reg: RegZ);
1759 MI.eraseFromParent();
1760 return true;
1761 }
1762 }
1763
1764 return false;
1765}
1766
1767MachineBasicBlock *
1768RISCVInstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
1769 assert(MI.getDesc().isBranch() && "Unexpected opcode!");
1770 // The branch target is always the last operand.
1771 int NumOp = MI.getNumExplicitOperands();
1772 return MI.getOperand(i: NumOp - 1).getMBB();
1773}
1774
1775bool RISCVInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
1776 int64_t BrOffset) const {
1777 unsigned XLen = STI.getXLen();
1778 // Ideally we could determine the supported branch offset from the
1779 // RISCVII::FormMask, but this can't be used for Pseudo instructions like
1780 // PseudoBR.
1781 switch (BranchOp) {
1782 default:
1783 llvm_unreachable("Unexpected opcode!");
1784 case RISCV::NDS_BBC:
1785 case RISCV::NDS_BBS:
1786 case RISCV::NDS_BEQC:
1787 case RISCV::NDS_BNEC:
1788 return isInt<11>(x: BrOffset);
1789 case RISCV::BEQ:
1790 case RISCV::BNE:
1791 case RISCV::BLT:
1792 case RISCV::BGE:
1793 case RISCV::BLTU:
1794 case RISCV::BGEU:
1795 case RISCV::BEQI:
1796 case RISCV::BNEI:
1797 case RISCV::CV_BEQIMM:
1798 case RISCV::CV_BNEIMM:
1799 case RISCV::QC_BEQI:
1800 case RISCV::QC_BNEI:
1801 case RISCV::QC_BGEI:
1802 case RISCV::QC_BLTI:
1803 case RISCV::QC_BLTUI:
1804 case RISCV::QC_BGEUI:
1805 case RISCV::QC_E_BEQI:
1806 case RISCV::QC_E_BNEI:
1807 case RISCV::QC_E_BGEI:
1808 case RISCV::QC_E_BLTI:
1809 case RISCV::QC_E_BLTUI:
1810 case RISCV::QC_E_BGEUI:
1811 return isInt<13>(x: BrOffset);
1812 case RISCV::JAL:
1813 case RISCV::PseudoBR:
1814 return isInt<21>(x: BrOffset);
1815 case RISCV::PseudoJump:
1816 return isInt<32>(x: SignExtend64(X: BrOffset + 0x800, B: XLen));
1817 }
1818}
1819
1820// If the operation has a predicated pseudo instruction, return the pseudo
1821// instruction opcode. Otherwise, return RISCV::INSTRUCTION_LIST_END.
1822// TODO: Support more operations.
1823unsigned getPredicatedOpcode(unsigned Opcode) {
1824 // clang-format off
1825 switch (Opcode) {
1826 case RISCV::ADD: return RISCV::PseudoCCADD;
1827 case RISCV::SUB: return RISCV::PseudoCCSUB;
1828 case RISCV::SLL: return RISCV::PseudoCCSLL;
1829 case RISCV::SRL: return RISCV::PseudoCCSRL;
1830 case RISCV::SRA: return RISCV::PseudoCCSRA;
1831 case RISCV::AND: return RISCV::PseudoCCAND;
1832 case RISCV::OR: return RISCV::PseudoCCOR;
1833 case RISCV::XOR: return RISCV::PseudoCCXOR;
1834 case RISCV::MAX: return RISCV::PseudoCCMAX;
1835 case RISCV::MAXU: return RISCV::PseudoCCMAXU;
1836 case RISCV::MIN: return RISCV::PseudoCCMIN;
1837 case RISCV::MINU: return RISCV::PseudoCCMINU;
1838 case RISCV::MUL: return RISCV::PseudoCCMUL;
1839 case RISCV::LUI: return RISCV::PseudoCCLUI;
1840 case RISCV::QC_LI: return RISCV::PseudoCCQC_LI;
1841 case RISCV::QC_E_LI: return RISCV::PseudoCCQC_E_LI;
1842
1843 case RISCV::ADDI: return RISCV::PseudoCCADDI;
1844 case RISCV::SLLI: return RISCV::PseudoCCSLLI;
1845 case RISCV::SRLI: return RISCV::PseudoCCSRLI;
1846 case RISCV::SRAI: return RISCV::PseudoCCSRAI;
1847 case RISCV::ANDI: return RISCV::PseudoCCANDI;
1848 case RISCV::ORI: return RISCV::PseudoCCORI;
1849 case RISCV::XORI: return RISCV::PseudoCCXORI;
1850
1851 case RISCV::ADDW: return RISCV::PseudoCCADDW;
1852 case RISCV::SUBW: return RISCV::PseudoCCSUBW;
1853 case RISCV::SLLW: return RISCV::PseudoCCSLLW;
1854 case RISCV::SRLW: return RISCV::PseudoCCSRLW;
1855 case RISCV::SRAW: return RISCV::PseudoCCSRAW;
1856
1857 case RISCV::ADDIW: return RISCV::PseudoCCADDIW;
1858 case RISCV::SLLIW: return RISCV::PseudoCCSLLIW;
1859 case RISCV::SRLIW: return RISCV::PseudoCCSRLIW;
1860 case RISCV::SRAIW: return RISCV::PseudoCCSRAIW;
1861
1862 case RISCV::ANDN: return RISCV::PseudoCCANDN;
1863 case RISCV::ORN: return RISCV::PseudoCCORN;
1864 case RISCV::XNOR: return RISCV::PseudoCCXNOR;
1865
1866 case RISCV::NDS_BFOS: return RISCV::PseudoCCNDS_BFOS;
1867 case RISCV::NDS_BFOZ: return RISCV::PseudoCCNDS_BFOZ;
1868 }
1869 // clang-format on
1870
1871 return RISCV::INSTRUCTION_LIST_END;
1872}
1873
1874/// Identify instructions that can be folded into a CCMOV instruction, and
1875/// return the defining instruction.
1876static MachineInstr *canFoldAsPredicatedOp(Register Reg,
1877 const MachineRegisterInfo &MRI,
1878 const TargetInstrInfo *TII,
1879 const RISCVSubtarget &STI) {
1880 if (!Reg.isVirtual())
1881 return nullptr;
1882 if (!MRI.hasOneNonDBGUse(RegNo: Reg))
1883 return nullptr;
1884 MachineInstr *MI = MRI.getVRegDef(Reg);
1885 if (!MI)
1886 return nullptr;
1887
1888 if (!STI.hasShortForwardBranchIMinMax() &&
1889 (MI->getOpcode() == RISCV::MAX || MI->getOpcode() == RISCV::MIN ||
1890 MI->getOpcode() == RISCV::MINU || MI->getOpcode() == RISCV::MAXU))
1891 return nullptr;
1892
1893 if (!STI.hasShortForwardBranchIMul() && MI->getOpcode() == RISCV::MUL)
1894 return nullptr;
1895
1896 // Check if MI can be predicated and folded into the CCMOV.
1897 if (getPredicatedOpcode(Opcode: MI->getOpcode()) == RISCV::INSTRUCTION_LIST_END)
1898 return nullptr;
1899 // Don't predicate li idiom.
1900 if (MI->getOpcode() == RISCV::ADDI && MI->getOperand(i: 1).isReg() &&
1901 MI->getOperand(i: 1).getReg() == RISCV::X0)
1902 return nullptr;
1903 // Check if MI has any other defs or physreg uses.
1904 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands())) {
1905 // Reject frame index operands, PEI can't handle the predicated pseudos.
1906 if (MO.isFI() || MO.isCPI() || MO.isJTI())
1907 return nullptr;
1908 if (!MO.isReg())
1909 continue;
1910 // MI can't have any tied operands, that would conflict with predication.
1911 if (MO.isTied())
1912 return nullptr;
1913 if (MO.isDef())
1914 return nullptr;
1915 // Allow constant physregs.
1916 if (MO.getReg().isPhysical() && !MRI.isConstantPhysReg(PhysReg: MO.getReg()))
1917 return nullptr;
1918 }
1919 bool DontMoveAcrossStores = true;
1920 if (!MI->isSafeToMove(SawStore&: DontMoveAcrossStores))
1921 return nullptr;
1922 return MI;
1923}
1924
1925MachineInstr *
1926RISCVInstrInfo::optimizeSelect(MachineInstr &MI,
1927 SmallPtrSetImpl<MachineInstr *> &SeenMIs,
1928 bool PreferFalse) const {
1929 assert(MI.getOpcode() == RISCV::PseudoCCMOVGPR &&
1930 "Unknown select instruction");
1931 if (!STI.hasShortForwardBranchIALU())
1932 return nullptr;
1933
1934 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
1935 MachineInstr *DefMI =
1936 canFoldAsPredicatedOp(Reg: MI.getOperand(i: 2).getReg(), MRI, TII: this, STI);
1937 bool Invert = !DefMI;
1938 if (!DefMI)
1939 DefMI = canFoldAsPredicatedOp(Reg: MI.getOperand(i: 1).getReg(), MRI, TII: this, STI);
1940 if (!DefMI)
1941 return nullptr;
1942
1943 // Find new register class to use.
1944 MachineOperand FalseReg = MI.getOperand(i: Invert ? 2 : 1);
1945 Register DestReg = MI.getOperand(i: 0).getReg();
1946 const TargetRegisterClass *PreviousClass = MRI.getRegClass(Reg: FalseReg.getReg());
1947 if (!MRI.constrainRegClass(Reg: DestReg, RC: PreviousClass))
1948 return nullptr;
1949
1950 unsigned PredOpc = getPredicatedOpcode(Opcode: DefMI->getOpcode());
1951 assert(PredOpc != RISCV::INSTRUCTION_LIST_END && "Unexpected opcode!");
1952
1953 // Create a new predicated version of DefMI.
1954 MachineInstrBuilder NewMI =
1955 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: PredOpc), DestReg);
1956
1957 // Copy the false register.
1958 NewMI.add(MO: FalseReg);
1959
1960 // Copy all the DefMI operands.
1961 const MCInstrDesc &DefDesc = DefMI->getDesc();
1962 for (unsigned i = 1, e = DefDesc.getNumOperands(); i != e; ++i)
1963 NewMI.add(MO: DefMI->getOperand(i));
1964
1965 // Add branch opcode, inverting if necessary.
1966 unsigned BCCOpcode = MI.getOperand(i: MI.getNumExplicitOperands() - 3).getImm();
1967 if (Invert)
1968 BCCOpcode = RISCVCC::getInverseBranchOpcode(BCC: BCCOpcode);
1969 NewMI.addImm(Val: BCCOpcode);
1970
1971 // Copy the condition portion.
1972 NewMI.add(MO: MI.getOperand(i: MI.getNumExplicitOperands() - 2));
1973 NewMI.add(MO: MI.getOperand(i: MI.getNumExplicitOperands() - 1));
1974
1975 // Update SeenMIs set: register newly created MI and erase removed DefMI.
1976 SeenMIs.insert(Ptr: NewMI);
1977 SeenMIs.erase(Ptr: DefMI);
1978
1979 // If MI is inside a loop, and DefMI is outside the loop, then kill flags on
1980 // DefMI would be invalid when transferred inside the loop. Checking for a
1981 // loop is expensive, but at least remove kill flags if they are in different
1982 // BBs.
1983 if (DefMI->getParent() != MI.getParent())
1984 NewMI->clearKillInfo();
1985
1986 // The caller will erase MI, but not DefMI.
1987 DefMI->eraseFromParent();
1988 return NewMI;
1989}
1990
1991unsigned RISCVInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
1992 if (MI.isMetaInstruction())
1993 return 0;
1994
1995 unsigned Opcode = MI.getOpcode();
1996
1997 if (Opcode == TargetOpcode::INLINEASM ||
1998 Opcode == TargetOpcode::INLINEASM_BR) {
1999 const MachineFunction &MF = *MI.getParent()->getParent();
2000 return getInlineAsmLength(Str: MI.getOperand(i: 0).getSymbolName(),
2001 MAI: MF.getTarget().getMCAsmInfo());
2002 }
2003
2004 if (requiresNTLHint(MI)) {
2005 if (STI.hasStdExtZca()) {
2006 if (isCompressibleInst(MI, STI))
2007 return 4; // c.ntl.all + c.load/c.store
2008 return 6; // c.ntl.all + load/store
2009 }
2010 return 8; // ntl.all + load/store
2011 }
2012
2013 if (Opcode == TargetOpcode::BUNDLE)
2014 return getInstBundleSize(MI);
2015
2016 if (MI.getParent() && MI.getParent()->getParent()) {
2017 if (isCompressibleInst(MI, STI))
2018 return 2;
2019 }
2020
2021 switch (Opcode) {
2022 case RISCV::PseudoMV_FPR16INX:
2023 case RISCV::PseudoMV_FPR32INX:
2024 case RISCV::PseudoClearGPR:
2025 // MV is always compressible to either c.mv or c.li rd, 0.
2026 return STI.hasStdExtZca() ? 2 : 4;
2027 // Below cases are for short forward branch pseudos
2028 case RISCV::PseudoCCMOVGPRNoX0:
2029 return get(Opcode: MI.getOperand(i: MI.getNumExplicitOperands() - 3).getImm())
2030 .getSize() +
2031 2;
2032 case RISCV::PseudoCCMOVGPR:
2033 case RISCV::PseudoCCADD:
2034 case RISCV::PseudoCCSUB:
2035 case RISCV::PseudoCCSLL:
2036 case RISCV::PseudoCCSRL:
2037 case RISCV::PseudoCCSRA:
2038 case RISCV::PseudoCCAND:
2039 case RISCV::PseudoCCOR:
2040 case RISCV::PseudoCCXOR:
2041 case RISCV::PseudoCCADDI:
2042 case RISCV::PseudoCCANDI:
2043 case RISCV::PseudoCCORI:
2044 case RISCV::PseudoCCXORI:
2045 case RISCV::PseudoCCLUI:
2046 case RISCV::PseudoCCSLLI:
2047 case RISCV::PseudoCCSRLI:
2048 case RISCV::PseudoCCSRAI:
2049 case RISCV::PseudoCCADDW:
2050 case RISCV::PseudoCCSUBW:
2051 case RISCV::PseudoCCSLLW:
2052 case RISCV::PseudoCCSRLW:
2053 case RISCV::PseudoCCSRAW:
2054 case RISCV::PseudoCCADDIW:
2055 case RISCV::PseudoCCSLLIW:
2056 case RISCV::PseudoCCSRLIW:
2057 case RISCV::PseudoCCSRAIW:
2058 case RISCV::PseudoCCANDN:
2059 case RISCV::PseudoCCORN:
2060 case RISCV::PseudoCCXNOR:
2061 case RISCV::PseudoCCMAX:
2062 case RISCV::PseudoCCMIN:
2063 case RISCV::PseudoCCMAXU:
2064 case RISCV::PseudoCCMINU:
2065 case RISCV::PseudoCCMUL:
2066 case RISCV::PseudoCCLB:
2067 case RISCV::PseudoCCLH:
2068 case RISCV::PseudoCCLW:
2069 case RISCV::PseudoCCLHU:
2070 case RISCV::PseudoCCLBU:
2071 case RISCV::PseudoCCLWU:
2072 case RISCV::PseudoCCLD:
2073 case RISCV::PseudoCCQC_LI:
2074 return get(Opcode: MI.getOperand(i: MI.getNumExplicitOperands() - 3).getImm())
2075 .getSize() +
2076 4;
2077 case RISCV::PseudoCCQC_E_LI:
2078 case RISCV::PseudoCCQC_E_LB:
2079 case RISCV::PseudoCCQC_E_LH:
2080 case RISCV::PseudoCCQC_E_LW:
2081 case RISCV::PseudoCCQC_E_LHU:
2082 case RISCV::PseudoCCQC_E_LBU:
2083 return get(Opcode: MI.getOperand(i: MI.getNumExplicitOperands() - 3).getImm())
2084 .getSize() +
2085 6;
2086 case TargetOpcode::STACKMAP:
2087 // The upper bound for a stackmap intrinsic is the full length of its shadow
2088 return StackMapOpers(&MI).getNumPatchBytes();
2089 case TargetOpcode::PATCHPOINT:
2090 // The size of the patchpoint intrinsic is the number of bytes requested
2091 return PatchPointOpers(&MI).getNumPatchBytes();
2092 case TargetOpcode::STATEPOINT: {
2093 // The size of the statepoint intrinsic is the number of bytes requested
2094 unsigned NumBytes = StatepointOpers(&MI).getNumPatchBytes();
2095 // No patch bytes means at most a PseudoCall is emitted
2096 return std::max(a: NumBytes, b: 8U);
2097 }
2098 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
2099 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
2100 case TargetOpcode::PATCHABLE_TAIL_CALL: {
2101 const MachineFunction &MF = *MI.getParent()->getParent();
2102 const Function &F = MF.getFunction();
2103 if (Opcode == TargetOpcode::PATCHABLE_FUNCTION_ENTER &&
2104 F.hasFnAttribute(Kind: "patchable-function-entry")) {
2105 unsigned Num =
2106 F.getFnAttributeAsParsedInteger(Kind: "patchable-function-entry");
2107 // Number of C.NOP or NOP
2108 return (STI.hasStdExtZca() ? 2 : 4) * Num;
2109 }
2110 // XRay uses C.JAL + 21 or 33 C.NOP for each sled in RV32 and RV64,
2111 // respectively.
2112 return STI.is64Bit() ? 68 : 44;
2113 }
2114 default:
2115 return get(Opcode).getSize();
2116 }
2117}
2118
2119bool RISCVInstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const {
2120 const unsigned Opcode = MI.getOpcode();
2121 switch (Opcode) {
2122 default:
2123 break;
2124 case RISCV::FSGNJ_D:
2125 case RISCV::FSGNJ_S:
2126 case RISCV::FSGNJ_H:
2127 case RISCV::FSGNJ_D_INX:
2128 case RISCV::FSGNJ_D_IN32X:
2129 case RISCV::FSGNJ_S_INX:
2130 case RISCV::FSGNJ_H_INX:
2131 // The canonical floating-point move is fsgnj rd, rs, rs.
2132 return MI.getOperand(i: 1).isReg() && MI.getOperand(i: 2).isReg() &&
2133 MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg();
2134 case RISCV::ADDI:
2135 case RISCV::ORI:
2136 case RISCV::XORI:
2137 return (MI.getOperand(i: 1).isReg() &&
2138 MI.getOperand(i: 1).getReg() == RISCV::X0) ||
2139 (MI.getOperand(i: 2).isImm() && MI.getOperand(i: 2).getImm() == 0);
2140 }
2141 return MI.isAsCheapAsAMove();
2142}
2143
2144std::optional<DestSourcePair>
2145RISCVInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
2146 if (MI.isMoveReg())
2147 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2148 switch (MI.getOpcode()) {
2149 default:
2150 break;
2151 case RISCV::ADD:
2152 case RISCV::OR:
2153 case RISCV::XOR:
2154 if (MI.getOperand(i: 1).isReg() && MI.getOperand(i: 1).getReg() == RISCV::X0 &&
2155 MI.getOperand(i: 2).isReg())
2156 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 2)};
2157 if (MI.getOperand(i: 2).isReg() && MI.getOperand(i: 2).getReg() == RISCV::X0 &&
2158 MI.getOperand(i: 1).isReg())
2159 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2160 break;
2161 case RISCV::ADDI:
2162 // Operand 1 can be a frameindex but callers expect registers
2163 if (MI.getOperand(i: 1).isReg() && MI.getOperand(i: 2).isImm() &&
2164 MI.getOperand(i: 2).getImm() == 0)
2165 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2166 break;
2167 case RISCV::SUB:
2168 if (MI.getOperand(i: 2).isReg() && MI.getOperand(i: 2).getReg() == RISCV::X0 &&
2169 MI.getOperand(i: 1).isReg())
2170 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2171 break;
2172 case RISCV::SH1ADD:
2173 case RISCV::SH1ADD_UW:
2174 case RISCV::SH2ADD:
2175 case RISCV::SH2ADD_UW:
2176 case RISCV::SH3ADD:
2177 case RISCV::SH3ADD_UW:
2178 if (MI.getOperand(i: 1).isReg() && MI.getOperand(i: 1).getReg() == RISCV::X0 &&
2179 MI.getOperand(i: 2).isReg())
2180 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 2)};
2181 break;
2182 case RISCV::FSGNJ_D:
2183 case RISCV::FSGNJ_S:
2184 case RISCV::FSGNJ_H:
2185 case RISCV::FSGNJ_D_INX:
2186 case RISCV::FSGNJ_D_IN32X:
2187 case RISCV::FSGNJ_S_INX:
2188 case RISCV::FSGNJ_H_INX:
2189 // The canonical floating-point move is fsgnj rd, rs, rs.
2190 if (MI.getOperand(i: 1).isReg() && MI.getOperand(i: 2).isReg() &&
2191 MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg())
2192 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2193 break;
2194 }
2195 return std::nullopt;
2196}
2197
2198MachineTraceStrategy RISCVInstrInfo::getMachineCombinerTraceStrategy() const {
2199 if (ForceMachineCombinerStrategy.getNumOccurrences() == 0) {
2200 // The option is unused. Choose Local strategy only for in-order cores. When
2201 // scheduling model is unspecified, use MinInstrCount strategy as more
2202 // generic one.
2203 const auto &SchedModel = STI.getSchedModel();
2204 return (!SchedModel.hasInstrSchedModel() || SchedModel.isOutOfOrder())
2205 ? MachineTraceStrategy::TS_MinInstrCount
2206 : MachineTraceStrategy::TS_Local;
2207 }
2208 // The strategy was forced by the option.
2209 return ForceMachineCombinerStrategy;
2210}
2211
2212void RISCVInstrInfo::finalizeInsInstrs(
2213 MachineInstr &Root, unsigned &Pattern,
2214 SmallVectorImpl<MachineInstr *> &InsInstrs) const {
2215 int16_t FrmOpIdx =
2216 RISCV::getNamedOperandIdx(Opcode: Root.getOpcode(), Name: RISCV::OpName::frm);
2217 if (FrmOpIdx < 0) {
2218 assert(all_of(InsInstrs,
2219 [](MachineInstr *MI) {
2220 return RISCV::getNamedOperandIdx(MI->getOpcode(),
2221 RISCV::OpName::frm) < 0;
2222 }) &&
2223 "New instructions require FRM whereas the old one does not have it");
2224 return;
2225 }
2226
2227 const MachineOperand &FRM = Root.getOperand(i: FrmOpIdx);
2228 MachineFunction &MF = *Root.getMF();
2229
2230 for (auto *NewMI : InsInstrs) {
2231 // We'd already added the FRM operand.
2232 if (static_cast<unsigned>(RISCV::getNamedOperandIdx(
2233 Opcode: NewMI->getOpcode(), Name: RISCV::OpName::frm)) != NewMI->getNumOperands())
2234 continue;
2235 MachineInstrBuilder MIB(MF, NewMI);
2236 MIB.add(MO: FRM);
2237 if (FRM.getImm() == RISCVFPRndMode::DYN)
2238 MIB.addUse(RegNo: RISCV::FRM, Flags: RegState::Implicit);
2239 }
2240}
2241
2242static bool isFADD(unsigned Opc) {
2243 switch (Opc) {
2244 default:
2245 return false;
2246 case RISCV::FADD_H:
2247 case RISCV::FADD_S:
2248 case RISCV::FADD_D:
2249 return true;
2250 }
2251}
2252
2253static bool isFSUB(unsigned Opc) {
2254 switch (Opc) {
2255 default:
2256 return false;
2257 case RISCV::FSUB_H:
2258 case RISCV::FSUB_S:
2259 case RISCV::FSUB_D:
2260 return true;
2261 }
2262}
2263
2264static bool isFMUL(unsigned Opc) {
2265 switch (Opc) {
2266 default:
2267 return false;
2268 case RISCV::FMUL_H:
2269 case RISCV::FMUL_S:
2270 case RISCV::FMUL_D:
2271 return true;
2272 }
2273}
2274
2275bool RISCVInstrInfo::isVectorAssociativeAndCommutative(const MachineInstr &Inst,
2276 bool Invert) const {
2277#define OPCODE_LMUL_CASE(OPC) \
2278 case RISCV::OPC##_M1: \
2279 case RISCV::OPC##_M2: \
2280 case RISCV::OPC##_M4: \
2281 case RISCV::OPC##_M8: \
2282 case RISCV::OPC##_MF2: \
2283 case RISCV::OPC##_MF4: \
2284 case RISCV::OPC##_MF8
2285
2286#define OPCODE_LMUL_MASK_CASE(OPC) \
2287 case RISCV::OPC##_M1_MASK: \
2288 case RISCV::OPC##_M2_MASK: \
2289 case RISCV::OPC##_M4_MASK: \
2290 case RISCV::OPC##_M8_MASK: \
2291 case RISCV::OPC##_MF2_MASK: \
2292 case RISCV::OPC##_MF4_MASK: \
2293 case RISCV::OPC##_MF8_MASK
2294
2295 unsigned Opcode = Inst.getOpcode();
2296 if (Invert) {
2297 if (auto InvOpcode = getInverseOpcode(Opcode))
2298 Opcode = *InvOpcode;
2299 else
2300 return false;
2301 }
2302
2303 // clang-format off
2304 switch (Opcode) {
2305 default:
2306 return false;
2307 OPCODE_LMUL_CASE(PseudoVADD_VV):
2308 OPCODE_LMUL_MASK_CASE(PseudoVADD_VV):
2309 OPCODE_LMUL_CASE(PseudoVMUL_VV):
2310 OPCODE_LMUL_MASK_CASE(PseudoVMUL_VV):
2311 return true;
2312 }
2313 // clang-format on
2314
2315#undef OPCODE_LMUL_MASK_CASE
2316#undef OPCODE_LMUL_CASE
2317}
2318
2319bool RISCVInstrInfo::areRVVInstsReassociable(const MachineInstr &Root,
2320 const MachineInstr &Prev) const {
2321 if (!areOpcodesEqualOrInverse(Opcode1: Root.getOpcode(), Opcode2: Prev.getOpcode()))
2322 return false;
2323
2324 assert(Root.getMF() == Prev.getMF());
2325 const MachineRegisterInfo *MRI = &Root.getMF()->getRegInfo();
2326 const TargetRegisterInfo *TRI = MRI->getTargetRegisterInfo();
2327
2328 // Make sure vtype operands are also the same.
2329 const MCInstrDesc &Desc = get(Opcode: Root.getOpcode());
2330 const uint64_t TSFlags = Desc.TSFlags;
2331
2332 auto checkImmOperand = [&](unsigned OpIdx) {
2333 return Root.getOperand(i: OpIdx).getImm() == Prev.getOperand(i: OpIdx).getImm();
2334 };
2335
2336 auto checkRegOperand = [&](unsigned OpIdx) {
2337 return Root.getOperand(i: OpIdx).getReg() == Prev.getOperand(i: OpIdx).getReg();
2338 };
2339
2340 // PassThru
2341 // TODO: Potentially we can loosen the condition to consider Root to be
2342 // associable with Prev if Root has NoReg as passthru. In which case we
2343 // also need to loosen the condition on vector policies between these.
2344 if (!checkRegOperand(1))
2345 return false;
2346
2347 // SEW
2348 if (RISCVII::hasSEWOp(TSFlags) &&
2349 !checkImmOperand(RISCVII::getSEWOpNum(Desc)))
2350 return false;
2351
2352 // Mask
2353 if (RISCVII::usesMaskPolicy(TSFlags)) {
2354 const MachineBasicBlock *MBB = Root.getParent();
2355 const MachineBasicBlock::const_reverse_iterator It1(&Root);
2356 const MachineBasicBlock::const_reverse_iterator It2(&Prev);
2357 Register MI1VReg;
2358
2359 bool SeenMI2 = false;
2360 for (auto End = MBB->rend(), It = It1; It != End; ++It) {
2361 if (It == It2) {
2362 SeenMI2 = true;
2363 if (!MI1VReg.isValid())
2364 // There is no V0 def between Root and Prev; they're sharing the
2365 // same V0.
2366 break;
2367 }
2368
2369 if (It->modifiesRegister(Reg: RISCV::V0, TRI)) {
2370 Register SrcReg = It->getOperand(i: 1).getReg();
2371 // If it's not VReg it'll be more difficult to track its defs, so
2372 // bailing out here just to be safe.
2373 if (!SrcReg.isVirtual())
2374 return false;
2375
2376 if (!MI1VReg.isValid()) {
2377 // This is the V0 def for Root.
2378 MI1VReg = SrcReg;
2379 continue;
2380 }
2381
2382 // Some random mask updates.
2383 if (!SeenMI2)
2384 continue;
2385
2386 // This is the V0 def for Prev; check if it's the same as that of
2387 // Root.
2388 if (MI1VReg != SrcReg)
2389 return false;
2390 else
2391 break;
2392 }
2393 }
2394
2395 // If we haven't encountered Prev, it's likely that this function was
2396 // called in a wrong way (e.g. Root is before Prev).
2397 assert(SeenMI2 && "Prev is expected to appear before Root");
2398 }
2399
2400 // Tail / Mask policies
2401 if (RISCVII::hasVecPolicyOp(TSFlags) &&
2402 !checkImmOperand(RISCVII::getVecPolicyOpNum(Desc)))
2403 return false;
2404
2405 // VL
2406 if (RISCVII::hasVLOp(TSFlags)) {
2407 unsigned OpIdx = RISCVII::getVLOpNum(Desc);
2408 const MachineOperand &Op1 = Root.getOperand(i: OpIdx);
2409 const MachineOperand &Op2 = Prev.getOperand(i: OpIdx);
2410 if (Op1.getType() != Op2.getType())
2411 return false;
2412 switch (Op1.getType()) {
2413 case MachineOperand::MO_Register:
2414 if (Op1.getReg() != Op2.getReg())
2415 return false;
2416 break;
2417 case MachineOperand::MO_Immediate:
2418 if (Op1.getImm() != Op2.getImm())
2419 return false;
2420 break;
2421 default:
2422 llvm_unreachable("Unrecognized VL operand type");
2423 }
2424 }
2425
2426 // Rounding modes
2427 if (int Idx = RISCVII::getFRMOpNum(Desc); Idx >= 0 && !checkImmOperand(Idx))
2428 return false;
2429 if (int Idx = RISCVII::getVXRMOpNum(Desc); Idx >= 0 && !checkImmOperand(Idx))
2430 return false;
2431
2432 return true;
2433}
2434
2435// Most of our RVV pseudos have passthru operand, so the real operands
2436// start from index = 2.
2437bool RISCVInstrInfo::hasReassociableVectorSibling(const MachineInstr &Inst,
2438 bool &Commuted) const {
2439 const MachineBasicBlock *MBB = Inst.getParent();
2440 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
2441 assert(RISCVII::isFirstDefTiedToFirstUse(get(Inst.getOpcode())) &&
2442 "Expect the present of passthrough operand.");
2443 MachineInstr *MI1 = MRI.getUniqueVRegDef(Reg: Inst.getOperand(i: 2).getReg());
2444 MachineInstr *MI2 = MRI.getUniqueVRegDef(Reg: Inst.getOperand(i: 3).getReg());
2445
2446 // If only one operand has the same or inverse opcode and it's the second
2447 // source operand, the operands must be commuted.
2448 Commuted = !areRVVInstsReassociable(Root: Inst, Prev: *MI1) &&
2449 areRVVInstsReassociable(Root: Inst, Prev: *MI2);
2450 if (Commuted)
2451 std::swap(a&: MI1, b&: MI2);
2452
2453 return areRVVInstsReassociable(Root: Inst, Prev: *MI1) &&
2454 (isVectorAssociativeAndCommutative(Inst: *MI1) ||
2455 isVectorAssociativeAndCommutative(Inst: *MI1, /* Invert */ true)) &&
2456 hasReassociableOperands(Inst: *MI1, MBB) &&
2457 MRI.hasOneNonDBGUse(RegNo: MI1->getOperand(i: 0).getReg());
2458}
2459
2460bool RISCVInstrInfo::hasReassociableOperands(
2461 const MachineInstr &Inst, const MachineBasicBlock *MBB) const {
2462 if (!isVectorAssociativeAndCommutative(Inst) &&
2463 !isVectorAssociativeAndCommutative(Inst, /*Invert=*/true))
2464 return TargetInstrInfo::hasReassociableOperands(Inst, MBB);
2465
2466 const MachineOperand &Op1 = Inst.getOperand(i: 2);
2467 const MachineOperand &Op2 = Inst.getOperand(i: 3);
2468 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
2469
2470 // We need virtual register definitions for the operands that we will
2471 // reassociate.
2472 MachineInstr *MI1 = nullptr;
2473 MachineInstr *MI2 = nullptr;
2474 if (Op1.isReg() && Op1.getReg().isVirtual())
2475 MI1 = MRI.getUniqueVRegDef(Reg: Op1.getReg());
2476 if (Op2.isReg() && Op2.getReg().isVirtual())
2477 MI2 = MRI.getUniqueVRegDef(Reg: Op2.getReg());
2478
2479 // And at least one operand must be defined in MBB.
2480 return MI1 && MI2 && (MI1->getParent() == MBB || MI2->getParent() == MBB);
2481}
2482
2483void RISCVInstrInfo::getReassociateOperandIndices(
2484 const MachineInstr &Root, unsigned Pattern,
2485 std::array<unsigned, 5> &OperandIndices) const {
2486 TargetInstrInfo::getReassociateOperandIndices(Root, Pattern, OperandIndices);
2487 if (RISCV::getRVVMCOpcode(RVVPseudoOpcode: Root.getOpcode())) {
2488 // Skip the passthrough operand, so increment all indices by one.
2489 for (unsigned I = 0; I < 5; ++I)
2490 ++OperandIndices[I];
2491 }
2492}
2493
2494bool RISCVInstrInfo::hasReassociableSibling(const MachineInstr &Inst,
2495 bool &Commuted) const {
2496 if (isVectorAssociativeAndCommutative(Inst) ||
2497 isVectorAssociativeAndCommutative(Inst, /*Invert=*/true))
2498 return hasReassociableVectorSibling(Inst, Commuted);
2499
2500 if (!TargetInstrInfo::hasReassociableSibling(Inst, Commuted))
2501 return false;
2502
2503 const MachineRegisterInfo &MRI = Inst.getMF()->getRegInfo();
2504 unsigned OperandIdx = Commuted ? 2 : 1;
2505 const MachineInstr &Sibling =
2506 *MRI.getVRegDef(Reg: Inst.getOperand(i: OperandIdx).getReg());
2507
2508 int16_t InstFrmOpIdx =
2509 RISCV::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name: RISCV::OpName::frm);
2510 int16_t SiblingFrmOpIdx =
2511 RISCV::getNamedOperandIdx(Opcode: Sibling.getOpcode(), Name: RISCV::OpName::frm);
2512
2513 return (InstFrmOpIdx < 0 && SiblingFrmOpIdx < 0) ||
2514 RISCV::hasEqualFRM(MI1: Inst, MI2: Sibling);
2515}
2516
2517bool RISCVInstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst,
2518 bool Invert) const {
2519 if (isVectorAssociativeAndCommutative(Inst, Invert))
2520 return true;
2521
2522 unsigned Opc = Inst.getOpcode();
2523 if (Invert) {
2524 auto InverseOpcode = getInverseOpcode(Opcode: Opc);
2525 if (!InverseOpcode)
2526 return false;
2527 Opc = *InverseOpcode;
2528 }
2529
2530 if (isFADD(Opc) || isFMUL(Opc))
2531 return Inst.getFlag(Flag: MachineInstr::MIFlag::FmReassoc) &&
2532 Inst.getFlag(Flag: MachineInstr::MIFlag::FmNsz);
2533
2534 switch (Opc) {
2535 default:
2536 return false;
2537 case RISCV::ADD:
2538 case RISCV::ADDW:
2539 case RISCV::AND:
2540 case RISCV::OR:
2541 case RISCV::XOR:
2542 // From RISC-V ISA spec, if both the high and low bits of the same product
2543 // are required, then the recommended code sequence is:
2544 //
2545 // MULH[[S]U] rdh, rs1, rs2
2546 // MUL rdl, rs1, rs2
2547 // (source register specifiers must be in same order and rdh cannot be the
2548 // same as rs1 or rs2)
2549 //
2550 // Microarchitectures can then fuse these into a single multiply operation
2551 // instead of performing two separate multiplies.
2552 // MachineCombiner may reassociate MUL operands and lose the fusion
2553 // opportunity.
2554 case RISCV::MUL:
2555 case RISCV::MULW:
2556 case RISCV::MIN:
2557 case RISCV::MINU:
2558 case RISCV::MAX:
2559 case RISCV::MAXU:
2560 case RISCV::FMIN_H:
2561 case RISCV::FMIN_S:
2562 case RISCV::FMIN_D:
2563 case RISCV::FMAX_H:
2564 case RISCV::FMAX_S:
2565 case RISCV::FMAX_D:
2566 return true;
2567 }
2568
2569 return false;
2570}
2571
2572std::optional<unsigned>
2573RISCVInstrInfo::getInverseOpcode(unsigned Opcode) const {
2574#define RVV_OPC_LMUL_CASE(OPC, INV) \
2575 case RISCV::OPC##_M1: \
2576 return RISCV::INV##_M1; \
2577 case RISCV::OPC##_M2: \
2578 return RISCV::INV##_M2; \
2579 case RISCV::OPC##_M4: \
2580 return RISCV::INV##_M4; \
2581 case RISCV::OPC##_M8: \
2582 return RISCV::INV##_M8; \
2583 case RISCV::OPC##_MF2: \
2584 return RISCV::INV##_MF2; \
2585 case RISCV::OPC##_MF4: \
2586 return RISCV::INV##_MF4; \
2587 case RISCV::OPC##_MF8: \
2588 return RISCV::INV##_MF8
2589
2590#define RVV_OPC_LMUL_MASK_CASE(OPC, INV) \
2591 case RISCV::OPC##_M1_MASK: \
2592 return RISCV::INV##_M1_MASK; \
2593 case RISCV::OPC##_M2_MASK: \
2594 return RISCV::INV##_M2_MASK; \
2595 case RISCV::OPC##_M4_MASK: \
2596 return RISCV::INV##_M4_MASK; \
2597 case RISCV::OPC##_M8_MASK: \
2598 return RISCV::INV##_M8_MASK; \
2599 case RISCV::OPC##_MF2_MASK: \
2600 return RISCV::INV##_MF2_MASK; \
2601 case RISCV::OPC##_MF4_MASK: \
2602 return RISCV::INV##_MF4_MASK; \
2603 case RISCV::OPC##_MF8_MASK: \
2604 return RISCV::INV##_MF8_MASK
2605
2606 switch (Opcode) {
2607 default:
2608 return std::nullopt;
2609 case RISCV::FADD_H:
2610 return RISCV::FSUB_H;
2611 case RISCV::FADD_S:
2612 return RISCV::FSUB_S;
2613 case RISCV::FADD_D:
2614 return RISCV::FSUB_D;
2615 case RISCV::FSUB_H:
2616 return RISCV::FADD_H;
2617 case RISCV::FSUB_S:
2618 return RISCV::FADD_S;
2619 case RISCV::FSUB_D:
2620 return RISCV::FADD_D;
2621 case RISCV::ADD:
2622 return RISCV::SUB;
2623 case RISCV::SUB:
2624 return RISCV::ADD;
2625 case RISCV::ADDW:
2626 return RISCV::SUBW;
2627 case RISCV::SUBW:
2628 return RISCV::ADDW;
2629 // clang-format off
2630 RVV_OPC_LMUL_CASE(PseudoVADD_VV, PseudoVSUB_VV);
2631 RVV_OPC_LMUL_MASK_CASE(PseudoVADD_VV, PseudoVSUB_VV);
2632 RVV_OPC_LMUL_CASE(PseudoVSUB_VV, PseudoVADD_VV);
2633 RVV_OPC_LMUL_MASK_CASE(PseudoVSUB_VV, PseudoVADD_VV);
2634 // clang-format on
2635 }
2636
2637#undef RVV_OPC_LMUL_MASK_CASE
2638#undef RVV_OPC_LMUL_CASE
2639}
2640
2641static bool canCombineFPFusedMultiply(const MachineInstr &Root,
2642 const MachineOperand &MO,
2643 bool DoRegPressureReduce) {
2644 if (!MO.isReg() || !MO.getReg().isVirtual())
2645 return false;
2646 const MachineRegisterInfo &MRI = Root.getMF()->getRegInfo();
2647 MachineInstr *MI = MRI.getVRegDef(Reg: MO.getReg());
2648 if (!MI || !isFMUL(Opc: MI->getOpcode()))
2649 return false;
2650
2651 if (!Root.getFlag(Flag: MachineInstr::MIFlag::FmContract) ||
2652 !MI->getFlag(Flag: MachineInstr::MIFlag::FmContract))
2653 return false;
2654
2655 // Try combining even if fmul has more than one use as it eliminates
2656 // dependency between fadd(fsub) and fmul. However, it can extend liveranges
2657 // for fmul operands, so reject the transformation in register pressure
2658 // reduction mode.
2659 if (DoRegPressureReduce && !MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()))
2660 return false;
2661
2662 // Do not combine instructions from different basic blocks.
2663 if (Root.getParent() != MI->getParent())
2664 return false;
2665 return RISCV::hasEqualFRM(MI1: Root, MI2: *MI);
2666}
2667
2668static bool getFPFusedMultiplyPatterns(MachineInstr &Root,
2669 SmallVectorImpl<unsigned> &Patterns,
2670 bool DoRegPressureReduce) {
2671 unsigned Opc = Root.getOpcode();
2672 bool IsFAdd = isFADD(Opc);
2673 if (!IsFAdd && !isFSUB(Opc))
2674 return false;
2675 bool Added = false;
2676 if (canCombineFPFusedMultiply(Root, MO: Root.getOperand(i: 1),
2677 DoRegPressureReduce)) {
2678 Patterns.push_back(Elt: IsFAdd ? RISCVMachineCombinerPattern::FMADD_AX
2679 : RISCVMachineCombinerPattern::FMSUB);
2680 Added = true;
2681 }
2682 if (canCombineFPFusedMultiply(Root, MO: Root.getOperand(i: 2),
2683 DoRegPressureReduce)) {
2684 Patterns.push_back(Elt: IsFAdd ? RISCVMachineCombinerPattern::FMADD_XA
2685 : RISCVMachineCombinerPattern::FNMSUB);
2686 Added = true;
2687 }
2688 return Added;
2689}
2690
2691static bool getFPPatterns(MachineInstr &Root,
2692 SmallVectorImpl<unsigned> &Patterns,
2693 bool DoRegPressureReduce) {
2694 return getFPFusedMultiplyPatterns(Root, Patterns, DoRegPressureReduce);
2695}
2696
2697/// Utility routine that checks if \param MO is defined by an
2698/// \param CombineOpc instruction in the basic block \param MBB
2699static const MachineInstr *canCombine(const MachineBasicBlock &MBB,
2700 const MachineOperand &MO,
2701 unsigned CombineOpc) {
2702 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2703 const MachineInstr *MI = nullptr;
2704
2705 if (MO.isReg() && MO.getReg().isVirtual())
2706 MI = MRI.getUniqueVRegDef(Reg: MO.getReg());
2707 // And it needs to be in the trace (otherwise, it won't have a depth).
2708 if (!MI || MI->getParent() != &MBB || MI->getOpcode() != CombineOpc)
2709 return nullptr;
2710 // Must only used by the user we combine with.
2711 if (!MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()))
2712 return nullptr;
2713
2714 return MI;
2715}
2716
2717/// Utility routine that checks if \param MO is defined by a SLLI in \param
2718/// MBB that can be combined by splitting across 2 SHXADD instructions. The
2719/// first SHXADD shift amount is given by \param OuterShiftAmt.
2720static bool canCombineShiftIntoShXAdd(const MachineBasicBlock &MBB,
2721 const MachineOperand &MO,
2722 unsigned OuterShiftAmt) {
2723 const MachineInstr *ShiftMI = canCombine(MBB, MO, CombineOpc: RISCV::SLLI);
2724 if (!ShiftMI)
2725 return false;
2726
2727 unsigned InnerShiftAmt = ShiftMI->getOperand(i: 2).getImm();
2728 if (InnerShiftAmt < OuterShiftAmt || (InnerShiftAmt - OuterShiftAmt) > 3)
2729 return false;
2730
2731 return true;
2732}
2733
2734// Returns the shift amount from a SHXADD instruction. Returns 0 if the
2735// instruction is not a SHXADD.
2736static unsigned getSHXADDShiftAmount(unsigned Opc) {
2737 switch (Opc) {
2738 default:
2739 return 0;
2740 case RISCV::SH1ADD:
2741 return 1;
2742 case RISCV::SH2ADD:
2743 return 2;
2744 case RISCV::SH3ADD:
2745 return 3;
2746 }
2747}
2748
2749// Returns the shift amount from a SHXADD.UW instruction. Returns 0 if the
2750// instruction is not a SHXADD.UW.
2751static unsigned getSHXADDUWShiftAmount(unsigned Opc) {
2752 switch (Opc) {
2753 default:
2754 return 0;
2755 case RISCV::SH1ADD_UW:
2756 return 1;
2757 case RISCV::SH2ADD_UW:
2758 return 2;
2759 case RISCV::SH3ADD_UW:
2760 return 3;
2761 }
2762}
2763
2764// Look for opportunities to combine (sh3add Z, (add X, (slli Y, 5))) into
2765// (sh3add (sh2add Y, Z), X).
2766static bool getSHXADDPatterns(const MachineInstr &Root,
2767 SmallVectorImpl<unsigned> &Patterns) {
2768 unsigned ShiftAmt = getSHXADDShiftAmount(Opc: Root.getOpcode());
2769 if (!ShiftAmt)
2770 return false;
2771
2772 const MachineBasicBlock &MBB = *Root.getParent();
2773
2774 const MachineInstr *AddMI = canCombine(MBB, MO: Root.getOperand(i: 2), CombineOpc: RISCV::ADD);
2775 if (!AddMI)
2776 return false;
2777
2778 bool Found = false;
2779 if (canCombineShiftIntoShXAdd(MBB, MO: AddMI->getOperand(i: 1), OuterShiftAmt: ShiftAmt)) {
2780 Patterns.push_back(Elt: RISCVMachineCombinerPattern::SHXADD_ADD_SLLI_OP1);
2781 Found = true;
2782 }
2783 if (canCombineShiftIntoShXAdd(MBB, MO: AddMI->getOperand(i: 2), OuterShiftAmt: ShiftAmt)) {
2784 Patterns.push_back(Elt: RISCVMachineCombinerPattern::SHXADD_ADD_SLLI_OP2);
2785 Found = true;
2786 }
2787
2788 return Found;
2789}
2790
2791CombinerObjective RISCVInstrInfo::getCombinerObjective(unsigned Pattern) const {
2792 switch (Pattern) {
2793 case RISCVMachineCombinerPattern::FMADD_AX:
2794 case RISCVMachineCombinerPattern::FMADD_XA:
2795 case RISCVMachineCombinerPattern::FMSUB:
2796 case RISCVMachineCombinerPattern::FNMSUB:
2797 return CombinerObjective::MustReduceDepth;
2798 default:
2799 return TargetInstrInfo::getCombinerObjective(Pattern);
2800 }
2801}
2802
2803bool RISCVInstrInfo::getMachineCombinerPatterns(
2804 MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns,
2805 bool DoRegPressureReduce) const {
2806
2807 if (getFPPatterns(Root, Patterns, DoRegPressureReduce))
2808 return true;
2809
2810 if (getSHXADDPatterns(Root, Patterns))
2811 return true;
2812
2813 return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
2814 DoRegPressureReduce);
2815}
2816
2817static unsigned getFPFusedMultiplyOpcode(unsigned RootOpc, unsigned Pattern) {
2818 switch (RootOpc) {
2819 default:
2820 llvm_unreachable("Unexpected opcode");
2821 case RISCV::FADD_H:
2822 return RISCV::FMADD_H;
2823 case RISCV::FADD_S:
2824 return RISCV::FMADD_S;
2825 case RISCV::FADD_D:
2826 return RISCV::FMADD_D;
2827 case RISCV::FSUB_H:
2828 return Pattern == RISCVMachineCombinerPattern::FMSUB ? RISCV::FMSUB_H
2829 : RISCV::FNMSUB_H;
2830 case RISCV::FSUB_S:
2831 return Pattern == RISCVMachineCombinerPattern::FMSUB ? RISCV::FMSUB_S
2832 : RISCV::FNMSUB_S;
2833 case RISCV::FSUB_D:
2834 return Pattern == RISCVMachineCombinerPattern::FMSUB ? RISCV::FMSUB_D
2835 : RISCV::FNMSUB_D;
2836 }
2837}
2838
2839static unsigned getAddendOperandIdx(unsigned Pattern) {
2840 switch (Pattern) {
2841 default:
2842 llvm_unreachable("Unexpected pattern");
2843 case RISCVMachineCombinerPattern::FMADD_AX:
2844 case RISCVMachineCombinerPattern::FMSUB:
2845 return 2;
2846 case RISCVMachineCombinerPattern::FMADD_XA:
2847 case RISCVMachineCombinerPattern::FNMSUB:
2848 return 1;
2849 }
2850}
2851
2852static void combineFPFusedMultiply(MachineInstr &Root, MachineInstr &Prev,
2853 unsigned Pattern,
2854 SmallVectorImpl<MachineInstr *> &InsInstrs,
2855 SmallVectorImpl<MachineInstr *> &DelInstrs) {
2856 MachineFunction *MF = Root.getMF();
2857 MachineRegisterInfo &MRI = MF->getRegInfo();
2858 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
2859
2860 MachineOperand &Mul1 = Prev.getOperand(i: 1);
2861 MachineOperand &Mul2 = Prev.getOperand(i: 2);
2862 MachineOperand &Dst = Root.getOperand(i: 0);
2863 MachineOperand &Addend = Root.getOperand(i: getAddendOperandIdx(Pattern));
2864
2865 Register DstReg = Dst.getReg();
2866 unsigned FusedOpc = getFPFusedMultiplyOpcode(RootOpc: Root.getOpcode(), Pattern);
2867 uint32_t IntersectedFlags = Root.getFlags() & Prev.getFlags();
2868 DebugLoc MergedLoc =
2869 DILocation::getMergedLocation(LocA: Root.getDebugLoc(), LocB: Prev.getDebugLoc());
2870
2871 bool Mul1IsKill = Mul1.isKill();
2872 bool Mul2IsKill = Mul2.isKill();
2873 bool AddendIsKill = Addend.isKill();
2874
2875 // We need to clear kill flags since we may be extending the live range past
2876 // a kill. If the mul had kill flags, we can preserve those since we know
2877 // where the previous range stopped.
2878 MRI.clearKillFlags(Reg: Mul1.getReg());
2879 MRI.clearKillFlags(Reg: Mul2.getReg());
2880
2881 MachineInstrBuilder MIB =
2882 BuildMI(MF&: *MF, MIMD: MergedLoc, MCID: TII->get(Opcode: FusedOpc), DestReg: DstReg)
2883 .addReg(RegNo: Mul1.getReg(), Flags: getKillRegState(B: Mul1IsKill))
2884 .addReg(RegNo: Mul2.getReg(), Flags: getKillRegState(B: Mul2IsKill))
2885 .addReg(RegNo: Addend.getReg(), Flags: getKillRegState(B: AddendIsKill))
2886 .setMIFlags(IntersectedFlags);
2887
2888 InsInstrs.push_back(Elt: MIB);
2889 if (MRI.hasOneNonDBGUse(RegNo: Prev.getOperand(i: 0).getReg()))
2890 DelInstrs.push_back(Elt: &Prev);
2891 DelInstrs.push_back(Elt: &Root);
2892}
2893
2894// Combine patterns like (sh3add Z, (add X, (slli Y, 5))) to
2895// (sh3add (sh2add Y, Z), X) if the shift amount can be split across two
2896// shXadd instructions. The outer shXadd keeps its original opcode.
2897static void
2898genShXAddAddShift(MachineInstr &Root, unsigned AddOpIdx,
2899 SmallVectorImpl<MachineInstr *> &InsInstrs,
2900 SmallVectorImpl<MachineInstr *> &DelInstrs,
2901 DenseMap<Register, unsigned> &InstrIdxForVirtReg) {
2902 MachineFunction *MF = Root.getMF();
2903 MachineRegisterInfo &MRI = MF->getRegInfo();
2904 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
2905
2906 unsigned OuterShiftAmt = getSHXADDShiftAmount(Opc: Root.getOpcode());
2907 assert(OuterShiftAmt != 0 && "Unexpected opcode");
2908
2909 MachineInstr *AddMI = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: 2).getReg());
2910 MachineInstr *ShiftMI =
2911 MRI.getUniqueVRegDef(Reg: AddMI->getOperand(i: AddOpIdx).getReg());
2912
2913 unsigned InnerShiftAmt = ShiftMI->getOperand(i: 2).getImm();
2914 assert(InnerShiftAmt >= OuterShiftAmt && "Unexpected shift amount");
2915
2916 unsigned InnerOpc;
2917 switch (InnerShiftAmt - OuterShiftAmt) {
2918 default:
2919 llvm_unreachable("Unexpected shift amount");
2920 case 0:
2921 InnerOpc = RISCV::ADD;
2922 break;
2923 case 1:
2924 InnerOpc = RISCV::SH1ADD;
2925 break;
2926 case 2:
2927 InnerOpc = RISCV::SH2ADD;
2928 break;
2929 case 3:
2930 InnerOpc = RISCV::SH3ADD;
2931 break;
2932 }
2933
2934 const MachineOperand &X = AddMI->getOperand(i: 3 - AddOpIdx);
2935 const MachineOperand &Y = ShiftMI->getOperand(i: 1);
2936 const MachineOperand &Z = Root.getOperand(i: 1);
2937
2938 Register NewVR = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2939
2940 auto MIB1 = BuildMI(MF&: *MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: InnerOpc), DestReg: NewVR)
2941 .addReg(RegNo: Y.getReg(), Flags: getKillRegState(B: Y.isKill()))
2942 .addReg(RegNo: Z.getReg(), Flags: getKillRegState(B: Z.isKill()));
2943 auto MIB2 = BuildMI(MF&: *MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: Root.getOpcode()),
2944 DestReg: Root.getOperand(i: 0).getReg())
2945 .addReg(RegNo: NewVR, Flags: RegState::Kill)
2946 .addReg(RegNo: X.getReg(), Flags: getKillRegState(B: X.isKill()));
2947
2948 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
2949 InsInstrs.push_back(Elt: MIB1);
2950 InsInstrs.push_back(Elt: MIB2);
2951 DelInstrs.push_back(Elt: ShiftMI);
2952 DelInstrs.push_back(Elt: AddMI);
2953 DelInstrs.push_back(Elt: &Root);
2954}
2955
2956void RISCVInstrInfo::genAlternativeCodeSequence(
2957 MachineInstr &Root, unsigned Pattern,
2958 SmallVectorImpl<MachineInstr *> &InsInstrs,
2959 SmallVectorImpl<MachineInstr *> &DelInstrs,
2960 DenseMap<Register, unsigned> &InstrIdxForVirtReg) const {
2961 MachineRegisterInfo &MRI = Root.getMF()->getRegInfo();
2962 switch (Pattern) {
2963 default:
2964 TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs,
2965 DelInstrs, InstIdxForVirtReg&: InstrIdxForVirtReg);
2966 return;
2967 case RISCVMachineCombinerPattern::FMADD_AX:
2968 case RISCVMachineCombinerPattern::FMSUB: {
2969 MachineInstr &Prev = *MRI.getVRegDef(Reg: Root.getOperand(i: 1).getReg());
2970 combineFPFusedMultiply(Root, Prev, Pattern, InsInstrs, DelInstrs);
2971 return;
2972 }
2973 case RISCVMachineCombinerPattern::FMADD_XA:
2974 case RISCVMachineCombinerPattern::FNMSUB: {
2975 MachineInstr &Prev = *MRI.getVRegDef(Reg: Root.getOperand(i: 2).getReg());
2976 combineFPFusedMultiply(Root, Prev, Pattern, InsInstrs, DelInstrs);
2977 return;
2978 }
2979 case RISCVMachineCombinerPattern::SHXADD_ADD_SLLI_OP1:
2980 genShXAddAddShift(Root, AddOpIdx: 1, InsInstrs, DelInstrs, InstrIdxForVirtReg);
2981 return;
2982 case RISCVMachineCombinerPattern::SHXADD_ADD_SLLI_OP2:
2983 genShXAddAddShift(Root, AddOpIdx: 2, InsInstrs, DelInstrs, InstrIdxForVirtReg);
2984 return;
2985 }
2986}
2987
2988bool RISCVInstrInfo::verifyInstruction(const MachineInstr &MI,
2989 StringRef &ErrInfo) const {
2990 MCInstrDesc const &Desc = MI.getDesc();
2991
2992 for (const auto &[Index, Operand] : enumerate(First: Desc.operands())) {
2993 const MachineOperand &MO = MI.getOperand(i: Index);
2994 unsigned OpType = Operand.OperandType;
2995 switch (OpType) {
2996 default:
2997 if (OpType >= RISCVOp::OPERAND_FIRST_RISCV_IMM &&
2998 OpType <= RISCVOp::OPERAND_LAST_RISCV_IMM) {
2999 if (!MO.isImm()) {
3000 ErrInfo = "Expected an immediate operand.";
3001 return false;
3002 }
3003 int64_t Imm = MO.getImm();
3004 bool Ok;
3005 switch (OpType) {
3006 default:
3007 llvm_unreachable("Unexpected operand type");
3008
3009#define CASE_OPERAND_UIMM(NUM) \
3010 case RISCVOp::OPERAND_UIMM##NUM: \
3011 Ok = isUInt<NUM>(Imm); \
3012 break;
3013#define CASE_OPERAND_UIMM_LSB_ZEROS(BITS, SUFFIX) \
3014 case RISCVOp::OPERAND_UIMM##BITS##_LSB##SUFFIX: { \
3015 constexpr size_t NumZeros = sizeof(#SUFFIX) - 1; \
3016 Ok = isShiftedUInt<BITS - NumZeros, NumZeros>(Imm); \
3017 break; \
3018 }
3019#define CASE_OPERAND_SIMM(NUM) \
3020 case RISCVOp::OPERAND_SIMM##NUM: \
3021 Ok = isInt<NUM>(Imm); \
3022 break;
3023 // clang-format off
3024 CASE_OPERAND_UIMM(1)
3025 CASE_OPERAND_UIMM(2)
3026 CASE_OPERAND_UIMM(3)
3027 CASE_OPERAND_UIMM(4)
3028 CASE_OPERAND_UIMM(5)
3029 CASE_OPERAND_UIMM(6)
3030 CASE_OPERAND_UIMM(7)
3031 CASE_OPERAND_UIMM(8)
3032 CASE_OPERAND_UIMM(9)
3033 CASE_OPERAND_UIMM(10)
3034 CASE_OPERAND_UIMM(12)
3035 CASE_OPERAND_UIMM(16)
3036 CASE_OPERAND_UIMM(32)
3037 CASE_OPERAND_UIMM(48)
3038 CASE_OPERAND_UIMM(64)
3039 CASE_OPERAND_UIMM_LSB_ZEROS(2, 0)
3040 CASE_OPERAND_UIMM_LSB_ZEROS(5, 0)
3041 CASE_OPERAND_UIMM_LSB_ZEROS(6, 0)
3042 CASE_OPERAND_UIMM_LSB_ZEROS(6, 000)
3043 CASE_OPERAND_UIMM_LSB_ZEROS(7, 00)
3044 CASE_OPERAND_UIMM_LSB_ZEROS(7, 000)
3045 CASE_OPERAND_UIMM_LSB_ZEROS(8, 00)
3046 CASE_OPERAND_UIMM_LSB_ZEROS(8, 000)
3047 CASE_OPERAND_UIMM_LSB_ZEROS(9, 000)
3048 // clang-format on
3049 case RISCVOp::OPERAND_UIMM4_PLUS1:
3050 Ok = Imm >= 1 && Imm <= 16;
3051 break;
3052 case RISCVOp::OPERAND_UIMM5_NONZERO:
3053 Ok = isUInt<5>(x: Imm) && (Imm != 0);
3054 break;
3055 case RISCVOp::OPERAND_UIMM5_GT3:
3056 Ok = isUInt<5>(x: Imm) && (Imm > 3);
3057 break;
3058 case RISCVOp::OPERAND_UIMM5_PLUS1:
3059 Ok = Imm >= 1 && Imm <= 32;
3060 break;
3061 case RISCVOp::OPERAND_UIMM6_PLUS1:
3062 Ok = Imm >= 1 && Imm <= 64;
3063 break;
3064 case RISCVOp::OPERAND_UIMM7_EQ_XLEN:
3065 Ok = Imm == STI.getXLen();
3066 break;
3067 case RISCVOp::OPERAND_UIMM8_GE32:
3068 Ok = isUInt<8>(x: Imm) && Imm >= 32;
3069 break;
3070 case RISCVOp::OPERAND_UIMM9_YBNDSWI:
3071 Ok = RISCV::isValidYBNDSWImm(Imm);
3072 break;
3073 case RISCVOp::OPERAND_SIMM10_LSB0000_NONZERO:
3074 Ok = isShiftedInt<6, 4>(x: Imm) && (Imm != 0);
3075 break;
3076 case RISCVOp::OPERAND_UIMM10_LSB00_NONZERO:
3077 Ok = isShiftedUInt<8, 2>(x: Imm) && (Imm != 0);
3078 break;
3079 case RISCVOp::OPERAND_UIMM16_NONZERO:
3080 Ok = isUInt<16>(x: Imm) && (Imm != 0);
3081 break;
3082 case RISCVOp::OPERAND_THREE:
3083 Ok = Imm == 3;
3084 break;
3085 case RISCVOp::OPERAND_FOUR:
3086 Ok = Imm == 4;
3087 break;
3088 case RISCVOp::OPERAND_IMM5_ZIBI:
3089 Ok = (isUInt<5>(x: Imm) && Imm != 0) || Imm == -1;
3090 break;
3091 // clang-format off
3092 CASE_OPERAND_SIMM(5)
3093 CASE_OPERAND_SIMM(6)
3094 CASE_OPERAND_SIMM(8)
3095 CASE_OPERAND_SIMM(10)
3096 CASE_OPERAND_SIMM(11)
3097 CASE_OPERAND_SIMM(12)
3098 CASE_OPERAND_SIMM(26)
3099 // clang-format on
3100 case RISCVOp::OPERAND_SIMM5_PLUS1:
3101 Ok = Imm >= -15 && Imm <= 16;
3102 break;
3103 case RISCVOp::OPERAND_SIMM5_NONZERO:
3104 Ok = isInt<5>(x: Imm) && (Imm != 0);
3105 break;
3106 case RISCVOp::OPERAND_SIMM6_NONZERO:
3107 Ok = Imm != 0 && isInt<6>(x: Imm);
3108 break;
3109 case RISCVOp::OPERAND_VTYPEI10:
3110 Ok = isUInt<10>(x: Imm) && RISCVVType::isValidVType(VType: Imm);
3111 break;
3112 case RISCVOp::OPERAND_VTYPEI11:
3113 Ok = isUInt<11>(x: Imm) && RISCVVType::isValidVType(VType: Imm);
3114 break;
3115 case RISCVOp::OPERAND_SIMM12_LSB00000:
3116 Ok = isShiftedInt<7, 5>(x: Imm);
3117 break;
3118 case RISCVOp::OPERAND_SIMM16_NONZERO:
3119 Ok = isInt<16>(x: Imm) && (Imm != 0);
3120 break;
3121 case RISCVOp::OPERAND_SIMM20_LI:
3122 Ok = isInt<20>(x: Imm);
3123 break;
3124 case RISCVOp::OPERAND_UIMMLOG2XLEN:
3125 Ok = STI.is64Bit() ? isUInt<6>(x: Imm) : isUInt<5>(x: Imm);
3126 break;
3127 case RISCVOp::OPERAND_UIMMLOG2XLEN_NONZERO:
3128 Ok = STI.is64Bit() ? isUInt<6>(x: Imm) : isUInt<5>(x: Imm);
3129 Ok = Ok && Imm != 0;
3130 break;
3131 case RISCVOp::OPERAND_CLUI_IMM:
3132 Ok = (isUInt<5>(x: Imm) && Imm != 0) || (Imm >= 0xfffe0 && Imm <= 0xfffff);
3133 break;
3134 case RISCVOp::OPERAND_RVKRNUM:
3135 Ok = Imm >= 0 && Imm <= 10;
3136 break;
3137 case RISCVOp::OPERAND_RVKRNUM_0_7:
3138 Ok = Imm >= 0 && Imm <= 7;
3139 break;
3140 case RISCVOp::OPERAND_RVKRNUM_1_10:
3141 Ok = Imm >= 1 && Imm <= 10;
3142 break;
3143 case RISCVOp::OPERAND_RVKRNUM_2_14:
3144 Ok = Imm >= 2 && Imm <= 14;
3145 break;
3146 case RISCVOp::OPERAND_RLIST:
3147 Ok = Imm >= RISCVZC::RA && Imm <= RISCVZC::RA_S0_S11;
3148 break;
3149 case RISCVOp::OPERAND_RLIST_S0:
3150 Ok = Imm >= RISCVZC::RA_S0 && Imm <= RISCVZC::RA_S0_S11;
3151 break;
3152 case RISCVOp::OPERAND_STACKADJ:
3153 Ok = Imm >= 0 && Imm <= 48 && Imm % 16 == 0;
3154 break;
3155 case RISCVOp::OPERAND_FRMARG:
3156 Ok = RISCVFPRndMode::isValidRoundingMode(Mode: Imm);
3157 break;
3158 case RISCVOp::OPERAND_RTZARG:
3159 Ok = Imm == RISCVFPRndMode::RTZ;
3160 break;
3161 case RISCVOp::OPERAND_SMTVType:
3162 Ok = XSMTVTypeMode::isValidSMTVTypeMode(Mode: Imm);
3163 break;
3164 case RISCVOp::OPERAND_SMTI8:
3165 Ok = Imm == XSMTVTypeMode::SMT_I8;
3166 break;
3167 case RISCVOp::OPERAND_COND_CODE:
3168 Ok = Imm >= 0 && Imm < RISCVCC::COND_INVALID;
3169 break;
3170 case RISCVOp::OPERAND_ATOMIC_ORDERING:
3171 Ok = isValidAtomicOrdering(I: Imm);
3172 break;
3173 case RISCVOp::OPERAND_VEC_POLICY:
3174 Ok = (Imm & (RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC)) ==
3175 Imm;
3176 break;
3177 case RISCVOp::OPERAND_SEW:
3178 Ok = (isUInt<5>(x: Imm) && RISCVVType::isValidSEW(SEW: 1 << Imm));
3179 break;
3180 case RISCVOp::OPERAND_SEW_MASK:
3181 Ok = Imm == 0;
3182 break;
3183 case RISCVOp::OPERAND_VEC_RM:
3184 assert(RISCVII::hasRoundModeOp(Desc.TSFlags));
3185 if (RISCVII::usesVXRM(TSFlags: Desc.TSFlags))
3186 Ok = isUInt<2>(x: Imm);
3187 else
3188 Ok = RISCVFPRndMode::isValidRoundingMode(Mode: Imm);
3189 break;
3190 case RISCVOp::OPERAND_XSFMM_VTYPE:
3191 Ok = RISCVVType::isValidXSfmmVType(VTypeI: Imm);
3192 break;
3193 case RISCVOp::OPERAND_XSFMM_TWIDEN:
3194 Ok = Imm == 1 || Imm == 2 || Imm == 4;
3195 break;
3196 }
3197 if (!Ok) {
3198 ErrInfo = "Invalid immediate";
3199 return false;
3200 }
3201 }
3202 break;
3203 case RISCVOp::OPERAND_SIMM12_LO:
3204 // TODO: We could be stricter about what non-register operands are
3205 // allowed.
3206 if (MO.isReg()) {
3207 ErrInfo = "Expected a non-register operand.";
3208 return false;
3209 }
3210 if (MO.isImm() && !isInt<12>(x: MO.getImm())) {
3211 ErrInfo = "Invalid immediate";
3212 return false;
3213 }
3214 break;
3215 case RISCVOp::OPERAND_UIMM20_LUI:
3216 case RISCVOp::OPERAND_UIMM20_AUIPC:
3217 // TODO: We could be stricter about what non-register operands are
3218 // allowed.
3219 if (MO.isReg()) {
3220 ErrInfo = "Expected a non-register operand.";
3221 return false;
3222 }
3223 if (MO.isImm() && !isUInt<20>(x: MO.getImm())) {
3224 ErrInfo = "Invalid immediate";
3225 return false;
3226 }
3227 break;
3228 case RISCVOp::OPERAND_BARE_SIMM32:
3229 // TODO: We could be stricter about what non-register operands are
3230 // allowed.
3231 if (MO.isReg()) {
3232 ErrInfo = "Expected a non-register operand.";
3233 return false;
3234 }
3235 if (MO.isImm() && !isInt<32>(x: MO.getImm())) {
3236 ErrInfo = "Invalid immediate";
3237 return false;
3238 }
3239 break;
3240 case RISCVOp::OPERAND_AVL:
3241 if (MO.isImm()) {
3242 int64_t Imm = MO.getImm();
3243 // VLMAX is represented as -1.
3244 if (!isUInt<5>(x: Imm) && Imm != -1) {
3245 ErrInfo = "Invalid immediate";
3246 return false;
3247 }
3248 } else if (!MO.isReg()) {
3249 ErrInfo = "Expected a register or immediate operand.";
3250 return false;
3251 }
3252 break;
3253 case RISCVOp::OPERAND_SFB_RHS:
3254 if (!MO.isReg() && !MO.isImm()) {
3255 ErrInfo = "Expected a register or immediate operand.";
3256 return false;
3257 }
3258 break;
3259 }
3260 }
3261
3262 const uint64_t TSFlags = Desc.TSFlags;
3263 if (RISCVII::hasVLOp(TSFlags)) {
3264 const MachineOperand &Op = MI.getOperand(i: RISCVII::getVLOpNum(Desc));
3265 if (!Op.isImm() && !Op.isReg()) {
3266 ErrInfo = "Invalid operand type for VL operand";
3267 return false;
3268 }
3269 if (Op.isReg() && Op.getReg().isValid()) {
3270 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
3271 auto *RC = MRI.getRegClass(Reg: Op.getReg());
3272 if (!RISCV::GPRNoX0RegClass.hasSubClassEq(RC)) {
3273 ErrInfo = "Invalid register class for VL operand";
3274 return false;
3275 }
3276 }
3277 if (!RISCVII::hasSEWOp(TSFlags)) {
3278 ErrInfo = "VL operand w/o SEW operand?";
3279 return false;
3280 }
3281 }
3282 if (RISCVII::hasSEWOp(TSFlags)) {
3283 unsigned OpIdx = RISCVII::getSEWOpNum(Desc);
3284 if (!MI.getOperand(i: OpIdx).isImm()) {
3285 ErrInfo = "SEW value expected to be an immediate";
3286 return false;
3287 }
3288 uint64_t Log2SEW = MI.getOperand(i: OpIdx).getImm();
3289 if (Log2SEW > 31) {
3290 ErrInfo = "Unexpected SEW value";
3291 return false;
3292 }
3293 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
3294 if (!RISCVVType::isValidSEW(SEW)) {
3295 ErrInfo = "Unexpected SEW value";
3296 return false;
3297 }
3298 }
3299 if (RISCVII::hasVecPolicyOp(TSFlags)) {
3300 unsigned OpIdx = RISCVII::getVecPolicyOpNum(Desc);
3301 if (!MI.getOperand(i: OpIdx).isImm()) {
3302 ErrInfo = "Policy operand expected to be an immediate";
3303 return false;
3304 }
3305 uint64_t Policy = MI.getOperand(i: OpIdx).getImm();
3306 if (Policy > (RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC)) {
3307 ErrInfo = "Invalid Policy Value";
3308 return false;
3309 }
3310 if (!RISCVII::hasVLOp(TSFlags)) {
3311 ErrInfo = "policy operand w/o VL operand?";
3312 return false;
3313 }
3314
3315 // VecPolicy operands can only exist on instructions with passthru/merge
3316 // arguments. Note that not all arguments with passthru have vec policy
3317 // operands- some instructions have implicit policies.
3318 unsigned UseOpIdx;
3319 if (!MI.isRegTiedToUseOperand(DefOpIdx: 0, UseOpIdx: &UseOpIdx)) {
3320 ErrInfo = "policy operand w/o tied operand?";
3321 return false;
3322 }
3323 }
3324
3325 if (int Idx = RISCVII::getFRMOpNum(Desc);
3326 Idx >= 0 && MI.getOperand(i: Idx).getImm() == RISCVFPRndMode::DYN &&
3327 !MI.readsRegister(Reg: RISCV::FRM, /*TRI=*/nullptr)) {
3328 ErrInfo = "dynamic rounding mode should read FRM";
3329 return false;
3330 }
3331
3332 return true;
3333}
3334
3335bool RISCVInstrInfo::canFoldIntoAddrMode(const MachineInstr &MemI, Register Reg,
3336 const MachineInstr &AddrI,
3337 ExtAddrMode &AM) const {
3338 switch (MemI.getOpcode()) {
3339 default:
3340 return false;
3341 case RISCV::LB:
3342 case RISCV::LBU:
3343 case RISCV::LH:
3344 case RISCV::LH_INX:
3345 case RISCV::LHU:
3346 case RISCV::LW:
3347 case RISCV::LW_INX:
3348 case RISCV::LWU:
3349 case RISCV::LD:
3350 case RISCV::LD_RV32:
3351 case RISCV::FLH:
3352 case RISCV::FLW:
3353 case RISCV::FLD:
3354 case RISCV::SB:
3355 case RISCV::SH:
3356 case RISCV::SH_INX:
3357 case RISCV::SW:
3358 case RISCV::SW_INX:
3359 case RISCV::SD:
3360 case RISCV::SD_RV32:
3361 case RISCV::FSH:
3362 case RISCV::FSW:
3363 case RISCV::FSD:
3364 break;
3365 }
3366
3367 if (MemI.getOperand(i: 0).getReg() == Reg)
3368 return false;
3369
3370 if (AddrI.getOpcode() != RISCV::ADDI || !AddrI.getOperand(i: 1).isReg() ||
3371 !AddrI.getOperand(i: 2).isImm())
3372 return false;
3373
3374 int64_t OldOffset = MemI.getOperand(i: 2).getImm();
3375 int64_t Disp = AddrI.getOperand(i: 2).getImm();
3376 int64_t NewOffset = OldOffset + Disp;
3377 if (!STI.is64Bit())
3378 NewOffset = SignExtend64<32>(x: NewOffset);
3379
3380 if (!isInt<12>(x: NewOffset))
3381 return false;
3382
3383 AM.BaseReg = AddrI.getOperand(i: 1).getReg();
3384 AM.ScaledReg = 0;
3385 AM.Scale = 0;
3386 AM.Displacement = NewOffset;
3387 AM.Form = ExtAddrMode::Formula::Basic;
3388 return true;
3389}
3390
3391MachineInstr *RISCVInstrInfo::emitLdStWithAddr(MachineInstr &MemI,
3392 const ExtAddrMode &AM) const {
3393
3394 const DebugLoc &DL = MemI.getDebugLoc();
3395 MachineBasicBlock &MBB = *MemI.getParent();
3396
3397 assert(AM.ScaledReg == 0 && AM.Scale == 0 &&
3398 "Addressing mode not supported for folding");
3399
3400 return BuildMI(BB&: MBB, I&: MemI, MIMD: DL, MCID: get(Opcode: MemI.getOpcode()))
3401 .addReg(RegNo: MemI.getOperand(i: 0).getReg(), Flags: getDefRegState(B: MemI.mayLoad()))
3402 .addReg(RegNo: AM.BaseReg)
3403 .addImm(Val: AM.Displacement)
3404 .setMemRefs(MemI.memoperands())
3405 .setMIFlags(MemI.getFlags());
3406}
3407
3408// TODO: At the moment, MIPS introduced paring of instructions operating with
3409// word or double word. This should be extended with more instructions when more
3410// vendors support load/store pairing.
3411bool RISCVInstrInfo::isPairableLdStInstOpc(unsigned Opc) {
3412 switch (Opc) {
3413 default:
3414 return false;
3415 case RISCV::SW:
3416 case RISCV::SD:
3417 case RISCV::LD:
3418 case RISCV::LW:
3419 return true;
3420 }
3421}
3422
3423bool RISCVInstrInfo::isLdStSafeToPair(const MachineInstr &LdSt,
3424 const TargetRegisterInfo *TRI) {
3425 // If this is a volatile load/store, don't mess with it.
3426 if (LdSt.hasOrderedMemoryRef() || LdSt.getNumExplicitOperands() != 3)
3427 return false;
3428
3429 if (LdSt.getOperand(i: 1).isFI())
3430 return true;
3431
3432 assert(LdSt.getOperand(1).isReg() && "Expected a reg operand.");
3433 // Can't cluster if the instruction modifies the base register
3434 // or it is update form. e.g. ld x5,8(x5)
3435 if (LdSt.modifiesRegister(Reg: LdSt.getOperand(i: 1).getReg(), TRI))
3436 return false;
3437
3438 if (!LdSt.getOperand(i: 2).isImm())
3439 return false;
3440
3441 return true;
3442}
3443
3444bool RISCVInstrInfo::getMemOperandsWithOffsetWidth(
3445 const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
3446 int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
3447 const TargetRegisterInfo *TRI) const {
3448 if (!LdSt.mayLoadOrStore())
3449 return false;
3450
3451 // Conservatively, only handle scalar loads/stores for now.
3452 switch (LdSt.getOpcode()) {
3453 case RISCV::LB:
3454 case RISCV::LBU:
3455 case RISCV::SB:
3456 case RISCV::LH:
3457 case RISCV::LH_INX:
3458 case RISCV::LHU:
3459 case RISCV::FLH:
3460 case RISCV::SH:
3461 case RISCV::SH_INX:
3462 case RISCV::FSH:
3463 case RISCV::LW:
3464 case RISCV::LW_INX:
3465 case RISCV::LWU:
3466 case RISCV::FLW:
3467 case RISCV::SW:
3468 case RISCV::SW_INX:
3469 case RISCV::FSW:
3470 case RISCV::LD:
3471 case RISCV::LD_RV32:
3472 case RISCV::FLD:
3473 case RISCV::SD:
3474 case RISCV::SD_RV32:
3475 case RISCV::FSD:
3476 break;
3477 default:
3478 return false;
3479 }
3480 const MachineOperand *BaseOp;
3481 OffsetIsScalable = false;
3482 if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width, TRI))
3483 return false;
3484 BaseOps.push_back(Elt: BaseOp);
3485 return true;
3486}
3487
3488// TODO: This was copied from SIInstrInfo. Could it be lifted to a common
3489// helper?
3490static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
3491 ArrayRef<const MachineOperand *> BaseOps1,
3492 const MachineInstr &MI2,
3493 ArrayRef<const MachineOperand *> BaseOps2) {
3494 // Only examine the first "base" operand of each instruction, on the
3495 // assumption that it represents the real base address of the memory access.
3496 // Other operands are typically offsets or indices from this base address.
3497 if (BaseOps1.front()->isIdenticalTo(Other: *BaseOps2.front()))
3498 return true;
3499
3500 if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
3501 return false;
3502
3503 auto MO1 = *MI1.memoperands_begin();
3504 auto MO2 = *MI2.memoperands_begin();
3505 if (MO1->getAddrSpace() != MO2->getAddrSpace())
3506 return false;
3507
3508 auto Base1 = MO1->getValue();
3509 auto Base2 = MO2->getValue();
3510 if (!Base1 || !Base2)
3511 return false;
3512 Base1 = getUnderlyingObject(V: Base1);
3513 Base2 = getUnderlyingObject(V: Base2);
3514
3515 if (isa<UndefValue>(Val: Base1) || isa<UndefValue>(Val: Base2))
3516 return false;
3517
3518 return Base1 == Base2;
3519}
3520
3521bool RISCVInstrInfo::shouldClusterMemOps(
3522 ArrayRef<const MachineOperand *> BaseOps1, int64_t Offset1,
3523 bool OffsetIsScalable1, ArrayRef<const MachineOperand *> BaseOps2,
3524 int64_t Offset2, bool OffsetIsScalable2, unsigned ClusterSize,
3525 unsigned NumBytes) const {
3526 // If the mem ops (to be clustered) do not have the same base ptr, then they
3527 // should not be clustered
3528 if (!BaseOps1.empty() && !BaseOps2.empty()) {
3529 const MachineInstr &FirstLdSt = *BaseOps1.front()->getParent();
3530 const MachineInstr &SecondLdSt = *BaseOps2.front()->getParent();
3531 if (!memOpsHaveSameBasePtr(MI1: FirstLdSt, BaseOps1, MI2: SecondLdSt, BaseOps2))
3532 return false;
3533 } else if (!BaseOps1.empty() || !BaseOps2.empty()) {
3534 // If only one base op is empty, they do not have the same base ptr
3535 return false;
3536 }
3537
3538 unsigned CacheLineSize =
3539 BaseOps1.front()->getParent()->getMF()->getSubtarget().getCacheLineSize();
3540 // Assume a cache line size of 64 bytes if no size is set in RISCVSubtarget.
3541 CacheLineSize = CacheLineSize ? CacheLineSize : 64;
3542 // Cluster if the memory operations are on the same or a neighbouring cache
3543 // line, but limit the maximum ClusterSize to avoid creating too much
3544 // additional register pressure.
3545 return ClusterSize <= 4 && std::abs(i: Offset1 - Offset2) < CacheLineSize;
3546}
3547
3548// Set BaseReg (the base register operand), Offset (the byte offset being
3549// accessed) and the access Width of the passed instruction that reads/writes
3550// memory. Returns false if the instruction does not read/write memory or the
3551// BaseReg/Offset/Width can't be determined. Is not guaranteed to always
3552// recognise base operands and offsets in all cases.
3553// TODO: Add an IsScalable bool ref argument (like the equivalent AArch64
3554// function) and set it as appropriate.
3555bool RISCVInstrInfo::getMemOperandWithOffsetWidth(
3556 const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset,
3557 LocationSize &Width, const TargetRegisterInfo *TRI) const {
3558 if (!LdSt.mayLoadOrStore())
3559 return false;
3560
3561 // Here we assume the standard RISC-V ISA, which uses a base+offset
3562 // addressing mode. You'll need to relax these conditions to support custom
3563 // load/store instructions.
3564 if (LdSt.getNumExplicitOperands() != 3)
3565 return false;
3566 if ((!LdSt.getOperand(i: 1).isReg() && !LdSt.getOperand(i: 1).isFI()) ||
3567 !LdSt.getOperand(i: 2).isImm())
3568 return false;
3569
3570 if (!LdSt.hasOneMemOperand())
3571 return false;
3572
3573 Width = (*LdSt.memoperands_begin())->getSize();
3574 BaseReg = &LdSt.getOperand(i: 1);
3575 Offset = LdSt.getOperand(i: 2).getImm();
3576 return true;
3577}
3578
3579bool RISCVInstrInfo::areMemAccessesTriviallyDisjoint(
3580 const MachineInstr &MIa, const MachineInstr &MIb) const {
3581 assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
3582 assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
3583
3584 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() ||
3585 MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
3586 return false;
3587
3588 // Retrieve the base register, offset from the base register and width. Width
3589 // is the size of memory that is being loaded/stored (e.g. 1, 2, 4). If
3590 // base registers are identical, and the offset of a lower memory access +
3591 // the width doesn't overlap the offset of a higher memory access,
3592 // then the memory accesses are different.
3593 const TargetRegisterInfo *TRI = STI.getRegisterInfo();
3594 const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
3595 int64_t OffsetA = 0, OffsetB = 0;
3596 LocationSize WidthA = LocationSize::precise(Value: 0),
3597 WidthB = LocationSize::precise(Value: 0);
3598 if (getMemOperandWithOffsetWidth(LdSt: MIa, BaseReg&: BaseOpA, Offset&: OffsetA, Width&: WidthA, TRI) &&
3599 getMemOperandWithOffsetWidth(LdSt: MIb, BaseReg&: BaseOpB, Offset&: OffsetB, Width&: WidthB, TRI)) {
3600 if (BaseOpA->isIdenticalTo(Other: *BaseOpB)) {
3601 int LowOffset = std::min(a: OffsetA, b: OffsetB);
3602 int HighOffset = std::max(a: OffsetA, b: OffsetB);
3603 LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
3604 if (LowWidth.hasValue() &&
3605 LowOffset + (int)LowWidth.getValue() <= HighOffset)
3606 return true;
3607 }
3608 }
3609 return false;
3610}
3611
3612std::pair<unsigned, unsigned>
3613RISCVInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
3614 const unsigned Mask = RISCVII::MO_DIRECT_FLAG_MASK;
3615 return std::make_pair(x: TF & Mask, y: TF & ~Mask);
3616}
3617
3618ArrayRef<std::pair<unsigned, const char *>>
3619RISCVInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
3620 using namespace RISCVII;
3621 static const std::pair<unsigned, const char *> TargetFlags[] = {
3622 {MO_CALL, "riscv-call"},
3623 {MO_LO, "riscv-lo"},
3624 {MO_HI, "riscv-hi"},
3625 {MO_PCREL_LO, "riscv-pcrel-lo"},
3626 {MO_PCREL_HI, "riscv-pcrel-hi"},
3627 {MO_GOT_HI, "riscv-got-hi"},
3628 {MO_TPREL_LO, "riscv-tprel-lo"},
3629 {MO_TPREL_HI, "riscv-tprel-hi"},
3630 {MO_TPREL_ADD, "riscv-tprel-add"},
3631 {MO_TLS_GOT_HI, "riscv-tls-got-hi"},
3632 {MO_TLS_GD_HI, "riscv-tls-gd-hi"},
3633 {MO_TLSDESC_HI, "riscv-tlsdesc-hi"},
3634 {MO_TLSDESC_LOAD_LO, "riscv-tlsdesc-load-lo"},
3635 {MO_TLSDESC_ADD_LO, "riscv-tlsdesc-add-lo"},
3636 {MO_TLSDESC_CALL, "riscv-tlsdesc-call"},
3637 {MO_QC_ACCESS, "riscv-qc-access"},
3638 };
3639 return ArrayRef(TargetFlags);
3640}
3641bool RISCVInstrInfo::isFunctionSafeToOutlineFrom(
3642 MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
3643 const Function &F = MF.getFunction();
3644
3645 // Can F be deduplicated by the linker? If it can, don't outline from it.
3646 if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage())
3647 return false;
3648
3649 // Don't outline from functions with section markings; the program could
3650 // expect that all the code is in the named section.
3651 if (F.hasSection())
3652 return false;
3653
3654 // It's safe to outline from MF.
3655 return true;
3656}
3657
3658bool RISCVInstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB,
3659 unsigned &Flags) const {
3660 // More accurate safety checking is done in getOutliningCandidateInfo.
3661 return TargetInstrInfo::isMBBSafeToOutlineFrom(MBB, Flags);
3662}
3663
3664// Enum values indicating how an outlined call should be constructed.
3665enum MachineOutlinerConstructionID {
3666 MachineOutlinerTailCall,
3667 MachineOutlinerDefault,
3668 MachineOutlinerRegSave
3669};
3670
3671bool RISCVInstrInfo::shouldOutlineFromFunctionByDefault(
3672 MachineFunction &MF) const {
3673 return MF.getFunction().hasMinSize();
3674}
3675
3676static bool isCandidatePatchable(const MachineBasicBlock &MBB) {
3677 const MachineFunction *MF = MBB.getParent();
3678 const Function &F = MF->getFunction();
3679 return F.getFnAttribute(Kind: "fentry-call").getValueAsBool() ||
3680 F.hasFnAttribute(Kind: "patchable-function-entry");
3681}
3682
3683static bool isMIReadsReg(const MachineInstr &MI, const TargetRegisterInfo *TRI,
3684 MCRegister RegNo) {
3685 return MI.readsRegister(Reg: RegNo, TRI) ||
3686 MI.getDesc().hasImplicitUseOfPhysReg(Reg: RegNo);
3687}
3688
3689static bool isMIModifiesReg(const MachineInstr &MI,
3690 const TargetRegisterInfo *TRI, MCRegister RegNo) {
3691 return MI.modifiesRegister(Reg: RegNo, TRI) ||
3692 MI.getDesc().hasImplicitDefOfPhysReg(Reg: RegNo);
3693}
3694
3695static bool cannotInsertTailCall(const MachineBasicBlock &MBB) {
3696 if (!MBB.back().isReturn())
3697 return true;
3698 if (isCandidatePatchable(MBB))
3699 return true;
3700
3701 // If the candidate reads the pre-set register
3702 // that can be used for expanding PseudoTAIL instruction,
3703 // then we cannot insert tail call.
3704 const TargetSubtargetInfo &STI = MBB.getParent()->getSubtarget();
3705 const RISCVMachineFunctionInfo *RVFI =
3706 MBB.getParent()->getInfo<RISCVMachineFunctionInfo>();
3707 // When cf-protection-branch is active, the outliner will emit PseudoTAILX7
3708 // which always uses X7. Otherwise, PseudoTAIL is emitted and the register
3709 // is determined by Zicfilp at encode time.
3710 MCRegister TailExpandUseRegNo =
3711 RVFI->hasCFProtectionBranch()
3712 ? RISCV::X7
3713 : RISCVII::getTailExpandUseRegNo(FeatureBits: STI.getFeatureBits());
3714 for (const MachineInstr &MI : MBB) {
3715 if (isMIReadsReg(MI, TRI: STI.getRegisterInfo(), RegNo: TailExpandUseRegNo))
3716 return true;
3717 if (isMIModifiesReg(MI, TRI: STI.getRegisterInfo(), RegNo: TailExpandUseRegNo))
3718 break;
3719 }
3720 return false;
3721}
3722
3723static Register findRegisterToSaveX5To(outliner::Candidate &C,
3724 const TargetRegisterInfo &TRI) {
3725 // Candidate registers for saving X5: t1-t6
3726 static const MCPhysReg TempRegs[] = {
3727 RISCV::X6, // t1
3728 RISCV::X7, // t2
3729 RISCV::X28, // t3
3730 RISCV::X29, // t4
3731 RISCV::X30, // t5
3732 RISCV::X31 // t6
3733 };
3734
3735 const MachineFunction *MF = C.getMF();
3736 const MachineRegisterInfo &MRI = MF->getRegInfo();
3737
3738 for (MCPhysReg Reg : TempRegs) {
3739 if (MRI.isReserved(PhysReg: Reg))
3740 continue;
3741
3742 if (C.isAvailableAcrossAndOutOfSeq(Reg, TRI) &&
3743 C.isAvailableInsideSeq(Reg, TRI)) {
3744 return Reg;
3745 }
3746 }
3747
3748 return Register();
3749}
3750
3751bool RISCVInstrInfo::analyzeCandidate(outliner::Candidate &C) const {
3752 // If the expansion register for tail calls is live across the candidate
3753 // outlined call site, we cannot outline that candidate as the expansion
3754 // would clobber the register.
3755 const RISCVMachineFunctionInfo *RVFI =
3756 C.getMF()->getInfo<RISCVMachineFunctionInfo>();
3757 MCRegister TailExpandUseReg =
3758 RVFI->hasCFProtectionBranch()
3759 ? RISCV::X7
3760 : RISCVII::getTailExpandUseRegNo(FeatureBits: STI.getFeatureBits());
3761 if (C.back().isReturn() &&
3762 !C.isAvailableAcrossAndOutOfSeq(Reg: TailExpandUseReg, TRI: RegInfo)) {
3763 LLVM_DEBUG(dbgs() << "MBB:\n" << *C.getMBB());
3764 LLVM_DEBUG(dbgs() << "Cannot be outlined between: " << C.front() << "and "
3765 << C.back());
3766 LLVM_DEBUG(dbgs() << "Because the tail-call register is live across "
3767 "the proposed outlined function call\n");
3768 return true;
3769 }
3770
3771 // If last instruction is return then we can rely on
3772 // the verification already performed in the getOutliningTypeImpl.
3773 if (C.back().isReturn()) {
3774 assert(!cannotInsertTailCall(*C.getMBB()) &&
3775 "The candidate who uses return instruction must be outlined "
3776 "using tail call");
3777 return false;
3778 }
3779
3780 // Filter out candidates where the X5 register (t0) can't be used to setup
3781 // the function call.
3782 if (!C.isAvailableInsideSeq(Reg: RISCV::X5, TRI: RegInfo))
3783 return true;
3784
3785 // If X5 is available in the region, use X5 directly (MachineOutlinerDefault).
3786 if (C.isAvailableAcrossAndOutOfSeq(Reg: RISCV::X5, TRI: RegInfo))
3787 return false;
3788
3789 // Otherwise, try to save X5 into t1-t6 (MachineOutlinerRegSave).
3790 if (OutlinerEnableRegSave && findRegisterToSaveX5To(C, TRI: RegInfo))
3791 return false;
3792
3793 return true;
3794}
3795
3796std::optional<std::unique_ptr<outliner::OutlinedFunction>>
3797RISCVInstrInfo::getOutliningCandidateInfo(
3798 const MachineModuleInfo &MMI,
3799 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
3800 unsigned MinRepeats) const {
3801
3802 // Analyze each candidate and erase the ones that are not viable.
3803 llvm::erase_if(C&: RepeatedSequenceLocs, P: [this](auto Candidate) {
3804 return analyzeCandidate(C&: Candidate);
3805 });
3806
3807 // If the sequence doesn't have enough candidates left, then we're done.
3808 if (RepeatedSequenceLocs.size() < MinRepeats)
3809 return std::nullopt;
3810
3811 // Each RepeatedSequenceLoc is identical.
3812 outliner::Candidate &Candidate = RepeatedSequenceLocs[0];
3813 unsigned InstrSizeCExt =
3814 Candidate.getMF()->getSubtarget<RISCVSubtarget>().hasStdExtZca() ? 2 : 4;
3815 unsigned CallOverhead = 0, FrameOverhead = 0;
3816
3817 // Count the number of CFI instructions in the candidate, if present.
3818 unsigned CFICount = 0;
3819 for (auto &I : Candidate) {
3820 if (I.isCFIInstruction())
3821 CFICount++;
3822 }
3823
3824 // Ensure CFI coverage matches: comparing the number of CFIs in the candidate
3825 // with the total number of CFIs in the parent function for each candidate.
3826 // Outlining only a subset of a function’s CFIs would split the unwind state
3827 // across two code regions and lead to incorrect address offsets between the
3828 // outlined body and the remaining code. To preserve correct unwind info, we
3829 // only outline when all CFIs in the function can be outlined together.
3830 for (outliner::Candidate &C : RepeatedSequenceLocs) {
3831 std::vector<MCCFIInstruction> CFIInstructions =
3832 C.getMF()->getFrameInstructions();
3833
3834 if (CFICount > 0 && CFICount != CFIInstructions.size())
3835 return std::nullopt;
3836 }
3837
3838 MachineOutlinerConstructionID MOCI = MachineOutlinerDefault;
3839 if (Candidate.back().isReturn()) {
3840 MOCI = MachineOutlinerTailCall;
3841 // tail call = auipc + jalr in the worst case without linker relaxation.
3842 // FIXME: This code suggests the JALR can be compressed - how?
3843 CallOverhead = 4 + InstrSizeCExt;
3844 // Using tail call we move ret instruction from caller to callee.
3845 FrameOverhead = 0;
3846 } else {
3847 // call t0, function = 8 bytes.
3848 CallOverhead = 8;
3849 // jr t0 = 4 bytes, 2 bytes if compressed instructions are enabled.
3850 FrameOverhead = InstrSizeCExt;
3851 }
3852
3853 // If we have CFI instructions, we can only outline if the outlined section
3854 // can be a tail call.
3855 if (MOCI != MachineOutlinerTailCall && CFICount > 0)
3856 return std::nullopt;
3857
3858 if (OutlinerEnableRegSave && MOCI == MachineOutlinerDefault) {
3859 // Set per-candidate overhead based on X5 availability
3860 for (auto &C : RepeatedSequenceLocs) {
3861
3862 if (C.isAvailableAcrossAndOutOfSeq(Reg: RISCV::X5, TRI: RegInfo)) {
3863 // X5 is available, just need the call
3864 unsigned CandCallOverhead = 8;
3865 C.setCallInfo(CID: MachineOutlinerDefault, CO: CandCallOverhead);
3866 } else {
3867 // X5 unavailable, need save + call + restore
3868 // Save (2-4) + Call (8) + Restore (2-4)
3869 unsigned CandCallOverhead = InstrSizeCExt + 8 + InstrSizeCExt;
3870 C.setCallInfo(CID: MachineOutlinerRegSave, CO: CandCallOverhead);
3871 }
3872 }
3873 } else {
3874 for (auto &C : RepeatedSequenceLocs)
3875 C.setCallInfo(CID: MOCI, CO: CallOverhead);
3876 }
3877
3878 unsigned SequenceSize = 0;
3879 for (auto &MI : Candidate)
3880 SequenceSize += getInstSizeInBytes(MI);
3881
3882 return std::make_unique<outliner::OutlinedFunction>(
3883 args&: RepeatedSequenceLocs, args&: SequenceSize, args&: FrameOverhead, args&: MOCI);
3884}
3885
3886outliner::InstrType
3887RISCVInstrInfo::getOutliningTypeImpl(const MachineModuleInfo &MMI,
3888 MachineBasicBlock::iterator &MBBI,
3889 unsigned Flags) const {
3890 MachineInstr &MI = *MBBI;
3891 MachineBasicBlock *MBB = MI.getParent();
3892 const TargetRegisterInfo *TRI =
3893 MBB->getParent()->getSubtarget().getRegisterInfo();
3894 const auto &F = MI.getMF()->getFunction();
3895
3896 // We can only outline CFI instructions if we will tail call the outlined
3897 // function, or fix up the CFI offsets. Currently, CFI instructions are
3898 // outlined only if in a tail call.
3899 if (MI.isCFIInstruction())
3900 return outliner::InstrType::Legal;
3901
3902 if (cannotInsertTailCall(MBB: *MBB) &&
3903 (MI.isReturn() || isMIModifiesReg(MI, TRI, RegNo: RISCV::X5)))
3904 return outliner::InstrType::Illegal;
3905
3906 // Make sure the operands don't reference something unsafe.
3907 for (const auto &MO : MI.operands()) {
3908
3909 // pcrel-hi and pcrel-lo can't put in separate sections, filter that out
3910 // if any possible.
3911 if (MO.getTargetFlags() == RISCVII::MO_PCREL_LO &&
3912 (MI.getMF()->getTarget().getFunctionSections() || F.hasComdat() ||
3913 F.hasSection() || F.getSectionPrefix()))
3914 return outliner::InstrType::Illegal;
3915 }
3916
3917 if (isLPAD(MI))
3918 return outliner::InstrType::Illegal;
3919
3920 return outliner::InstrType::Legal;
3921}
3922
3923void RISCVInstrInfo::buildOutlinedFrame(
3924 MachineBasicBlock &MBB, MachineFunction &MF,
3925 const outliner::OutlinedFunction &OF) const {
3926
3927 if (OF.FrameConstructionID == MachineOutlinerTailCall)
3928 return;
3929
3930 MBB.addLiveIn(PhysReg: RISCV::X5);
3931
3932 // Add in a return instruction to the end of the outlined frame.
3933 MBB.insert(I: MBB.end(), MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: RISCV::JALR))
3934 .addReg(RegNo: RISCV::X0, Flags: RegState::Define)
3935 .addReg(RegNo: RISCV::X5)
3936 .addImm(Val: 0));
3937}
3938
3939MachineBasicBlock::iterator RISCVInstrInfo::insertOutlinedCall(
3940 Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It,
3941 MachineFunction &MF, outliner::Candidate &C) const {
3942
3943 if (C.CallConstructionID == MachineOutlinerTailCall) {
3944 const RISCVMachineFunctionInfo *RVFI =
3945 MF.getInfo<RISCVMachineFunctionInfo>();
3946 unsigned TailOpc =
3947 RVFI->hasCFProtectionBranch() ? RISCV::PseudoTAILX7 : RISCV::PseudoTAIL;
3948 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: TailOpc))
3949 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName()),
3950 /*Offset=*/0, TargetFlags: RISCVII::MO_CALL));
3951 return It;
3952 }
3953
3954 if (C.CallConstructionID == MachineOutlinerRegSave) {
3955 Register SaveReg = findRegisterToSaveX5To(C, TRI: RegInfo);
3956 assert(SaveReg && "Cannot find an available register to save/restore X5.");
3957
3958 // Save: ADDI SaveReg, X5, 0 (equivalent to MV SaveReg, X5)
3959 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: RISCV::ADDI), DestReg: SaveReg)
3960 .addReg(RegNo: RISCV::X5)
3961 .addImm(Val: 0));
3962 It++;
3963
3964 // Call: PseudoCALLReg X5
3965 It = MBB.insert(
3966 I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: RISCV::PseudoCALLReg), DestReg: RISCV::X5)
3967 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName()), Offset: 0,
3968 TargetFlags: RISCVII::MO_CALL));
3969 MachineBasicBlock::iterator CallPt = It;
3970 It++;
3971
3972 // Restore: ADDI X5, SaveReg, 0 (equivalent to MV X5, SaveReg)
3973 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: RISCV::ADDI), DestReg: RISCV::X5)
3974 .addReg(RegNo: SaveReg)
3975 .addImm(Val: 0));
3976
3977 return CallPt;
3978 }
3979
3980 // Add in a call instruction to the outlined function at the given location.
3981 It = MBB.insert(I: It,
3982 MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: RISCV::PseudoCALLReg), DestReg: RISCV::X5)
3983 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName()), Offset: 0,
3984 TargetFlags: RISCVII::MO_CALL));
3985 return It;
3986}
3987
3988void RISCVInstrInfo::buildClearRegister(Register Reg, MachineBasicBlock &MBB,
3989 MachineBasicBlock::iterator Iter,
3990 DebugLoc &DL,
3991 bool AllowSideEffects) const {
3992
3993 const MachineFunction &MF = *MBB.getParent();
3994 const RISCVRegisterInfo &TRI = *STI.getRegisterInfo();
3995
3996 if (TRI.isGeneralPurposeRegister(MF, Reg)) {
3997 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: RISCV::PseudoClearGPR), DestReg: Reg);
3998 } else if (RISCV::FPR32RegClass.contains(Reg)) {
3999 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: RISCV::PseudoClearFPR32), DestReg: Reg);
4000 } else if (RISCV::FPR64RegClass.contains(Reg)) {
4001 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: RISCV::PseudoClearFPR64), DestReg: Reg);
4002 } else if (RISCV::FPR128RegClass.contains(Reg)) {
4003 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: RISCV::PseudoClearFPR128), DestReg: Reg);
4004 } else if (RISCV::VRRegClass.contains(Reg)) {
4005 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: RISCV::PseudoClearVR), DestReg: Reg);
4006 } else {
4007 llvm::reportFatalInternalError(
4008 reason: "buildClearRegister is not implemented for " + TRI.getRegAsmName(Reg));
4009 }
4010}
4011
4012std::optional<RegImmPair> RISCVInstrInfo::isAddImmediate(const MachineInstr &MI,
4013 Register Reg) const {
4014 // TODO: Handle cases where Reg is a super- or sub-register of the
4015 // destination register.
4016 const MachineOperand &Op0 = MI.getOperand(i: 0);
4017 if (!Op0.isReg() || Reg != Op0.getReg())
4018 return std::nullopt;
4019
4020 // Don't consider ADDIW as a candidate because the caller may not be aware
4021 // of its sign extension behaviour.
4022 if (MI.getOpcode() == RISCV::ADDI && MI.getOperand(i: 1).isReg() &&
4023 MI.getOperand(i: 2).isImm())
4024 return RegImmPair{MI.getOperand(i: 1).getReg(), MI.getOperand(i: 2).getImm()};
4025
4026 return std::nullopt;
4027}
4028
4029// MIR printer helper function to annotate Operands with a comment.
4030std::string RISCVInstrInfo::createMIROperandComment(
4031 const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx,
4032 const TargetRegisterInfo *TRI) const {
4033 // Print a generic comment for this operand if there is one.
4034 std::string GenericComment =
4035 TargetInstrInfo::createMIROperandComment(MI, Op, OpIdx, TRI);
4036 if (!GenericComment.empty())
4037 return GenericComment;
4038
4039 const MCInstrDesc &Desc = MI.getDesc();
4040 if (OpIdx >= Desc.getNumOperands())
4041 return std::string();
4042
4043 std::string Comment;
4044 raw_string_ostream OS(Comment);
4045
4046 const MCOperandInfo &OpInfo = Desc.operands()[OpIdx];
4047
4048 // Print the full VType operand of vsetvli/vsetivli instructions, and the SEW
4049 // operand of vector codegen pseudos.
4050 switch (OpInfo.OperandType) {
4051 case RISCVOp::OPERAND_VTYPEI10:
4052 case RISCVOp::OPERAND_VTYPEI11: {
4053 unsigned Imm = Op.getImm();
4054 RISCVVType::printVType(VType: Imm, OS);
4055 break;
4056 }
4057 case RISCVOp::OPERAND_XSFMM_VTYPE: {
4058 unsigned Imm = Op.getImm();
4059 RISCVVType::printXSfmmVType(VType: Imm, OS);
4060 break;
4061 }
4062 case RISCVOp::OPERAND_XSFMM_TWIDEN: {
4063 unsigned Imm = Op.getImm();
4064 OS << "w" << Imm;
4065 break;
4066 }
4067 case RISCVOp::OPERAND_SEW:
4068 case RISCVOp::OPERAND_SEW_MASK: {
4069 unsigned Log2SEW = Op.getImm();
4070 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
4071 assert(RISCVVType::isValidSEW(SEW) && "Unexpected SEW");
4072 OS << "e" << SEW;
4073 break;
4074 }
4075 case RISCVOp::OPERAND_VEC_POLICY: {
4076 unsigned Policy = Op.getImm();
4077 assert(Policy <= (RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC) &&
4078 "Invalid Policy Value");
4079 OS << (Policy & RISCVVType::TAIL_AGNOSTIC ? "ta" : "tu") << ", "
4080 << (Policy & RISCVVType::MASK_AGNOSTIC ? "ma" : "mu");
4081 break;
4082 }
4083 case RISCVOp::OPERAND_AVL:
4084 if (Op.isImm() && Op.getImm() == -1)
4085 OS << "vl=VLMAX";
4086 else
4087 OS << "vl";
4088 break;
4089 case RISCVOp::OPERAND_VEC_RM:
4090 if (RISCVII::usesVXRM(TSFlags: Desc.TSFlags)) {
4091 assert(RISCVVXRndMode::isValidRoundingMode(Op.getImm()));
4092 auto VXRM = static_cast<RISCVVXRndMode::RoundingMode>(Op.getImm());
4093 OS << "vxrm=" << RISCVVXRndMode::roundingModeToString(RndMode: VXRM);
4094 } else {
4095 assert(RISCVFPRndMode::isValidRoundingMode(Op.getImm()));
4096 auto FRM = static_cast<RISCVFPRndMode::RoundingMode>(Op.getImm());
4097 OS << "frm=" << RISCVFPRndMode::roundingModeToString(RndMode: FRM);
4098 }
4099 break;
4100 }
4101
4102 return Comment;
4103}
4104
4105// clang-format off
4106#define CASE_RVV_OPCODE_UNMASK_LMUL(OP, LMUL) \
4107 RISCV::Pseudo##OP##_##LMUL
4108
4109#define CASE_RVV_OPCODE_MASK_LMUL(OP, LMUL) \
4110 RISCV::Pseudo##OP##_##LMUL##_MASK
4111
4112#define CASE_RVV_OPCODE_LMUL(OP, LMUL) \
4113 CASE_RVV_OPCODE_UNMASK_LMUL(OP, LMUL): \
4114 case CASE_RVV_OPCODE_MASK_LMUL(OP, LMUL)
4115
4116#define CASE_RVV_OPCODE_UNMASK_WIDEN(OP) \
4117 CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF8): \
4118 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF4): \
4119 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF2): \
4120 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M1): \
4121 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M2): \
4122 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M4)
4123
4124#define CASE_RVV_OPCODE_UNMASK(OP) \
4125 CASE_RVV_OPCODE_UNMASK_WIDEN(OP): \
4126 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M8)
4127
4128#define CASE_RVV_OPCODE_MASK_WIDEN(OP) \
4129 CASE_RVV_OPCODE_MASK_LMUL(OP, MF8): \
4130 case CASE_RVV_OPCODE_MASK_LMUL(OP, MF4): \
4131 case CASE_RVV_OPCODE_MASK_LMUL(OP, MF2): \
4132 case CASE_RVV_OPCODE_MASK_LMUL(OP, M1): \
4133 case CASE_RVV_OPCODE_MASK_LMUL(OP, M2): \
4134 case CASE_RVV_OPCODE_MASK_LMUL(OP, M4)
4135
4136#define CASE_RVV_OPCODE_MASK(OP) \
4137 CASE_RVV_OPCODE_MASK_WIDEN(OP): \
4138 case CASE_RVV_OPCODE_MASK_LMUL(OP, M8)
4139
4140#define CASE_RVV_OPCODE_WIDEN(OP) \
4141 CASE_RVV_OPCODE_UNMASK_WIDEN(OP): \
4142 case CASE_RVV_OPCODE_MASK_WIDEN(OP)
4143
4144#define CASE_RVV_OPCODE(OP) \
4145 CASE_RVV_OPCODE_UNMASK(OP): \
4146 case CASE_RVV_OPCODE_MASK(OP)
4147// clang-format on
4148
4149// clang-format off
4150#define CASE_VMA_OPCODE_COMMON(OP, TYPE, LMUL) \
4151 RISCV::PseudoV##OP##_##TYPE##_##LMUL
4152
4153#define CASE_VMA_OPCODE_LMULS(OP, TYPE) \
4154 CASE_VMA_OPCODE_COMMON(OP, TYPE, MF8): \
4155 case CASE_VMA_OPCODE_COMMON(OP, TYPE, MF4): \
4156 case CASE_VMA_OPCODE_COMMON(OP, TYPE, MF2): \
4157 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M1): \
4158 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M2): \
4159 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M4): \
4160 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M8)
4161
4162// VFMA instructions are SEW specific.
4163#define CASE_VFMA_OPCODE_COMMON(OP, TYPE, LMUL, SEW) \
4164 RISCV::PseudoV##OP##_##TYPE##_##LMUL##_##SEW
4165
4166#define CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE, SEW) \
4167 CASE_VFMA_OPCODE_COMMON(OP, TYPE, M1, SEW): \
4168 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M2, SEW): \
4169 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M4, SEW): \
4170 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M8, SEW)
4171
4172#define CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE, SEW) \
4173 CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF2, SEW): \
4174 case CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE, SEW)
4175
4176#define CASE_VFMA_OPCODE_LMULS_MF4(OP, TYPE, SEW) \
4177 CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF4, SEW): \
4178 case CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE, SEW)
4179
4180#define CASE_VFMA_OPCODE_VV(OP) \
4181 CASE_VFMA_OPCODE_LMULS_MF4(OP, VV, E16): \
4182 case CASE_VFMA_OPCODE_LMULS_MF4(OP##_ALT, VV, E16): \
4183 case CASE_VFMA_OPCODE_LMULS_MF2(OP, VV, E32): \
4184 case CASE_VFMA_OPCODE_LMULS_M1(OP, VV, E64)
4185
4186#define CASE_VFMA_SPLATS(OP) \
4187 CASE_VFMA_OPCODE_LMULS_MF4(OP, VFPR16, E16): \
4188 case CASE_VFMA_OPCODE_LMULS_MF4(OP##_ALT, VFPR16, E16): \
4189 case CASE_VFMA_OPCODE_LMULS_MF2(OP, VFPR32, E32): \
4190 case CASE_VFMA_OPCODE_LMULS_M1(OP, VFPR64, E64)
4191// clang-format on
4192
4193bool RISCVInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
4194 unsigned &SrcOpIdx1,
4195 unsigned &SrcOpIdx2) const {
4196 const MCInstrDesc &Desc = MI.getDesc();
4197 if (!Desc.isCommutable())
4198 return false;
4199
4200 switch (MI.getOpcode()) {
4201 case RISCV::TH_MVEQZ:
4202 case RISCV::TH_MVNEZ:
4203 // We can't commute operands if operand 2 (i.e., rs1 in
4204 // mveqz/mvnez rd,rs1,rs2) is the zero-register (as it is
4205 // not valid as the in/out-operand 1).
4206 if (MI.getOperand(i: 2).getReg() == RISCV::X0)
4207 return false;
4208 // Operands 1 and 2 are commutable, if we switch the opcode.
4209 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 1, CommutableOpIdx2: 2);
4210 case RISCV::QC_SELECTIEQ:
4211 case RISCV::QC_SELECTINE:
4212 case RISCV::QC_SELECTIIEQ:
4213 case RISCV::QC_SELECTIINE:
4214 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 1, CommutableOpIdx2: 2);
4215 case RISCV::QC_MVEQ:
4216 case RISCV::QC_MVNE:
4217 case RISCV::QC_MVLT:
4218 case RISCV::QC_MVGE:
4219 case RISCV::QC_MVLTU:
4220 case RISCV::QC_MVGEU:
4221 case RISCV::QC_MVEQI:
4222 case RISCV::QC_MVNEI:
4223 case RISCV::QC_MVLTI:
4224 case RISCV::QC_MVGEI:
4225 case RISCV::QC_MVLTUI:
4226 case RISCV::QC_MVGEUI:
4227 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 1, CommutableOpIdx2: 4);
4228 case RISCV::TH_MULA:
4229 case RISCV::TH_MULAW:
4230 case RISCV::TH_MULAH:
4231 case RISCV::TH_MULS:
4232 case RISCV::TH_MULSW:
4233 case RISCV::TH_MULSH:
4234 // Operands 2 and 3 are commutable.
4235 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 2, CommutableOpIdx2: 3);
4236 case RISCV::PseudoCCMOVGPRNoX0:
4237 case RISCV::PseudoCCMOVGPR:
4238 // Operands 1 and 2 are commutable.
4239 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 1, CommutableOpIdx2: 2);
4240 case CASE_RVV_OPCODE(VADD_VV):
4241 case CASE_RVV_OPCODE(VAND_VV):
4242 case CASE_RVV_OPCODE(VOR_VV):
4243 case CASE_RVV_OPCODE(VXOR_VV):
4244 case CASE_RVV_OPCODE_MASK(VMSEQ_VV):
4245 case CASE_RVV_OPCODE_MASK(VMSNE_VV):
4246 case CASE_RVV_OPCODE(VMIN_VV):
4247 case CASE_RVV_OPCODE(VMINU_VV):
4248 case CASE_RVV_OPCODE(VMAX_VV):
4249 case CASE_RVV_OPCODE(VMAXU_VV):
4250 case CASE_RVV_OPCODE(VMUL_VV):
4251 case CASE_RVV_OPCODE(VMULH_VV):
4252 case CASE_RVV_OPCODE(VMULHU_VV):
4253 case CASE_RVV_OPCODE_WIDEN(VWADD_VV):
4254 case CASE_RVV_OPCODE_WIDEN(VWADDU_VV):
4255 case CASE_RVV_OPCODE_WIDEN(VWMUL_VV):
4256 case CASE_RVV_OPCODE_WIDEN(VWMULU_VV):
4257 case CASE_RVV_OPCODE_WIDEN(VWMACC_VV):
4258 case CASE_RVV_OPCODE_WIDEN(VWMACCU_VV):
4259 case CASE_RVV_OPCODE(VABD_VV):
4260 case CASE_RVV_OPCODE(VABDU_VV):
4261 case CASE_RVV_OPCODE_WIDEN(VWABDA_VV):
4262 case CASE_RVV_OPCODE_WIDEN(VWABDAU_VV):
4263 case CASE_RVV_OPCODE_UNMASK(VADC_VVM):
4264 case CASE_RVV_OPCODE(VSADD_VV):
4265 case CASE_RVV_OPCODE(VSADDU_VV):
4266 case CASE_RVV_OPCODE(VAADD_VV):
4267 case CASE_RVV_OPCODE(VAADDU_VV):
4268 case CASE_RVV_OPCODE(VSMUL_VV):
4269 case CASE_RVV_OPCODE_LMUL(VDOT4A_VV, MF2):
4270 case CASE_RVV_OPCODE_LMUL(VDOT4A_VV, M1):
4271 case CASE_RVV_OPCODE_LMUL(VDOT4A_VV, M2):
4272 case CASE_RVV_OPCODE_LMUL(VDOT4A_VV, M4):
4273 case CASE_RVV_OPCODE_LMUL(VDOT4A_VV, M8):
4274 case CASE_RVV_OPCODE_LMUL(VDOT4AU_VV, MF2):
4275 case CASE_RVV_OPCODE_LMUL(VDOT4AU_VV, M1):
4276 case CASE_RVV_OPCODE_LMUL(VDOT4AU_VV, M2):
4277 case CASE_RVV_OPCODE_LMUL(VDOT4AU_VV, M4):
4278 case CASE_RVV_OPCODE_LMUL(VDOT4AU_VV, M8):
4279 // Operands 2 and 3 are commutable.
4280 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 2, CommutableOpIdx2: 3);
4281 case CASE_VFMA_SPLATS(FMADD):
4282 case CASE_VFMA_SPLATS(FMSUB):
4283 case CASE_VFMA_SPLATS(FMACC):
4284 case CASE_VFMA_SPLATS(FMSAC):
4285 case CASE_VFMA_SPLATS(FNMADD):
4286 case CASE_VFMA_SPLATS(FNMSUB):
4287 case CASE_VFMA_SPLATS(FNMACC):
4288 case CASE_VFMA_SPLATS(FNMSAC):
4289 case CASE_VFMA_OPCODE_VV(FMACC):
4290 case CASE_VFMA_OPCODE_VV(FMSAC):
4291 case CASE_VFMA_OPCODE_VV(FNMACC):
4292 case CASE_VFMA_OPCODE_VV(FNMSAC):
4293 case CASE_VMA_OPCODE_LMULS(MADD, VX):
4294 case CASE_VMA_OPCODE_LMULS(NMSUB, VX):
4295 case CASE_VMA_OPCODE_LMULS(MACC, VX):
4296 case CASE_VMA_OPCODE_LMULS(NMSAC, VX):
4297 case CASE_VMA_OPCODE_LMULS(MACC, VV):
4298 case CASE_VMA_OPCODE_LMULS(NMSAC, VV): {
4299 // If the tail policy is undisturbed we can't commute.
4300 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags));
4301 if ((MI.getOperand(i: RISCVII::getVecPolicyOpNum(Desc: MI.getDesc())).getImm() &
4302 1) == 0)
4303 return false;
4304
4305 // For these instructions we can only swap operand 1 and operand 3 by
4306 // changing the opcode.
4307 unsigned CommutableOpIdx1 = 1;
4308 unsigned CommutableOpIdx2 = 3;
4309 if (!fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1,
4310 CommutableOpIdx2))
4311 return false;
4312 return true;
4313 }
4314 case CASE_VFMA_OPCODE_VV(FMADD):
4315 case CASE_VFMA_OPCODE_VV(FMSUB):
4316 case CASE_VFMA_OPCODE_VV(FNMADD):
4317 case CASE_VFMA_OPCODE_VV(FNMSUB):
4318 case CASE_VMA_OPCODE_LMULS(MADD, VV):
4319 case CASE_VMA_OPCODE_LMULS(NMSUB, VV): {
4320 // If the tail policy is undisturbed we can't commute.
4321 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags));
4322 if ((MI.getOperand(i: RISCVII::getVecPolicyOpNum(Desc: MI.getDesc())).getImm() &
4323 1) == 0)
4324 return false;
4325
4326 // For these instructions we have more freedom. We can commute with the
4327 // other multiplicand or with the addend/subtrahend/minuend.
4328
4329 // Any fixed operand must be from source 1, 2 or 3.
4330 if (SrcOpIdx1 != CommuteAnyOperandIndex && SrcOpIdx1 > 3)
4331 return false;
4332 if (SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx2 > 3)
4333 return false;
4334
4335 // It both ops are fixed one must be the tied source.
4336 if (SrcOpIdx1 != CommuteAnyOperandIndex &&
4337 SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx1 != 1 && SrcOpIdx2 != 1)
4338 return false;
4339
4340 // Look for two different register operands assumed to be commutable
4341 // regardless of the FMA opcode. The FMA opcode is adjusted later if
4342 // needed.
4343 if (SrcOpIdx1 == CommuteAnyOperandIndex ||
4344 SrcOpIdx2 == CommuteAnyOperandIndex) {
4345 // At least one of operands to be commuted is not specified and
4346 // this method is free to choose appropriate commutable operands.
4347 unsigned CommutableOpIdx1 = SrcOpIdx1;
4348 if (SrcOpIdx1 == SrcOpIdx2) {
4349 // Both of operands are not fixed. Set one of commutable
4350 // operands to the tied source.
4351 CommutableOpIdx1 = 1;
4352 } else if (SrcOpIdx1 == CommuteAnyOperandIndex) {
4353 // Only one of the operands is not fixed.
4354 CommutableOpIdx1 = SrcOpIdx2;
4355 }
4356
4357 // CommutableOpIdx1 is well defined now. Let's choose another commutable
4358 // operand and assign its index to CommutableOpIdx2.
4359 unsigned CommutableOpIdx2;
4360 if (CommutableOpIdx1 != 1) {
4361 // If we haven't already used the tied source, we must use it now.
4362 CommutableOpIdx2 = 1;
4363 } else {
4364 Register Op1Reg = MI.getOperand(i: CommutableOpIdx1).getReg();
4365
4366 // The commuted operands should have different registers.
4367 // Otherwise, the commute transformation does not change anything and
4368 // is useless. We use this as a hint to make our decision.
4369 if (Op1Reg != MI.getOperand(i: 2).getReg())
4370 CommutableOpIdx2 = 2;
4371 else
4372 CommutableOpIdx2 = 3;
4373 }
4374
4375 // Assign the found pair of commutable indices to SrcOpIdx1 and
4376 // SrcOpIdx2 to return those values.
4377 if (!fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1,
4378 CommutableOpIdx2))
4379 return false;
4380 }
4381
4382 return true;
4383 }
4384 }
4385
4386 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
4387}
4388
4389// clang-format off
4390#define CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL) \
4391 case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL: \
4392 Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL; \
4393 break;
4394
4395#define CASE_VMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE) \
4396 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF8) \
4397 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4) \
4398 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2) \
4399 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1) \
4400 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2) \
4401 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4) \
4402 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8)
4403
4404// VFMA depends on SEW.
4405#define CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL, SEW) \
4406 case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL##_##SEW: \
4407 Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL##_##SEW; \
4408 break;
4409
4410#define CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE, SEW) \
4411 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1, SEW) \
4412 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2, SEW) \
4413 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4, SEW) \
4414 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8, SEW)
4415
4416#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE, SEW) \
4417 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2, SEW) \
4418 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE, SEW)
4419
4420#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE, SEW) \
4421 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4, SEW) \
4422 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE, SEW)
4423
4424#define CASE_VFMA_CHANGE_OPCODE_VV(OLDOP, NEWOP) \
4425 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VV, E16) \
4426 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP##_ALT, NEWOP##_ALT, VV, E16) \
4427 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VV, E32) \
4428 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VV, E64)
4429
4430#define CASE_VFMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP) \
4431 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VFPR16, E16) \
4432 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP##_ALT, NEWOP##_ALT, VFPR16, E16) \
4433 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VFPR32, E32) \
4434 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VFPR64, E64)
4435// clang-format on
4436
4437MachineInstr *RISCVInstrInfo::commuteInstructionImpl(MachineInstr &MI,
4438 bool NewMI,
4439 unsigned OpIdx1,
4440 unsigned OpIdx2) const {
4441 auto cloneIfNew = [NewMI](MachineInstr &MI) -> MachineInstr & {
4442 if (NewMI)
4443 return *MI.getParent()->getParent()->CloneMachineInstr(Orig: &MI);
4444 return MI;
4445 };
4446
4447 switch (MI.getOpcode()) {
4448 case RISCV::TH_MVEQZ:
4449 case RISCV::TH_MVNEZ: {
4450 auto &WorkingMI = cloneIfNew(MI);
4451 WorkingMI.setDesc(get(Opcode: MI.getOpcode() == RISCV::TH_MVEQZ ? RISCV::TH_MVNEZ
4452 : RISCV::TH_MVEQZ));
4453 return TargetInstrInfo::commuteInstructionImpl(MI&: WorkingMI, NewMI: false, OpIdx1,
4454 OpIdx2);
4455 }
4456 case RISCV::QC_SELECTIEQ:
4457 case RISCV::QC_SELECTINE:
4458 case RISCV::QC_SELECTIIEQ:
4459 case RISCV::QC_SELECTIINE:
4460 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
4461 case RISCV::QC_MVEQ:
4462 case RISCV::QC_MVNE:
4463 case RISCV::QC_MVLT:
4464 case RISCV::QC_MVGE:
4465 case RISCV::QC_MVLTU:
4466 case RISCV::QC_MVGEU:
4467 case RISCV::QC_MVEQI:
4468 case RISCV::QC_MVNEI:
4469 case RISCV::QC_MVLTI:
4470 case RISCV::QC_MVGEI:
4471 case RISCV::QC_MVLTUI:
4472 case RISCV::QC_MVGEUI: {
4473 auto &WorkingMI = cloneIfNew(MI);
4474 WorkingMI.setDesc(get(Opcode: getInverseXqcicmOpcode(Opcode: MI.getOpcode())));
4475 return TargetInstrInfo::commuteInstructionImpl(MI&: WorkingMI, NewMI: false, OpIdx1,
4476 OpIdx2);
4477 }
4478 case RISCV::PseudoCCMOVGPRNoX0:
4479 case RISCV::PseudoCCMOVGPR: {
4480 // CCMOV can be commuted by inverting the condition.
4481 unsigned BCC = MI.getOperand(i: MI.getNumExplicitOperands() - 3).getImm();
4482 BCC = RISCVCC::getInverseBranchOpcode(BCC);
4483 auto &WorkingMI = cloneIfNew(MI);
4484 WorkingMI.getOperand(i: MI.getNumExplicitOperands() - 3).setImm(BCC);
4485 return TargetInstrInfo::commuteInstructionImpl(MI&: WorkingMI, /*NewMI*/ false,
4486 OpIdx1, OpIdx2);
4487 }
4488 case CASE_VFMA_SPLATS(FMACC):
4489 case CASE_VFMA_SPLATS(FMADD):
4490 case CASE_VFMA_SPLATS(FMSAC):
4491 case CASE_VFMA_SPLATS(FMSUB):
4492 case CASE_VFMA_SPLATS(FNMACC):
4493 case CASE_VFMA_SPLATS(FNMADD):
4494 case CASE_VFMA_SPLATS(FNMSAC):
4495 case CASE_VFMA_SPLATS(FNMSUB):
4496 case CASE_VFMA_OPCODE_VV(FMACC):
4497 case CASE_VFMA_OPCODE_VV(FMSAC):
4498 case CASE_VFMA_OPCODE_VV(FNMACC):
4499 case CASE_VFMA_OPCODE_VV(FNMSAC):
4500 case CASE_VMA_OPCODE_LMULS(MADD, VX):
4501 case CASE_VMA_OPCODE_LMULS(NMSUB, VX):
4502 case CASE_VMA_OPCODE_LMULS(MACC, VX):
4503 case CASE_VMA_OPCODE_LMULS(NMSAC, VX):
4504 case CASE_VMA_OPCODE_LMULS(MACC, VV):
4505 case CASE_VMA_OPCODE_LMULS(NMSAC, VV): {
4506 // It only make sense to toggle these between clobbering the
4507 // addend/subtrahend/minuend one of the multiplicands.
4508 assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");
4509 assert((OpIdx1 == 3 || OpIdx2 == 3) && "Unexpected opcode index");
4510 unsigned Opc;
4511 switch (MI.getOpcode()) {
4512 default:
4513 llvm_unreachable("Unexpected opcode");
4514 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMACC, FMADD)
4515 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMADD, FMACC)
4516 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMSAC, FMSUB)
4517 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMSUB, FMSAC)
4518 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMACC, FNMADD)
4519 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMADD, FNMACC)
4520 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMSAC, FNMSUB)
4521 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMSUB, FNMSAC)
4522 CASE_VFMA_CHANGE_OPCODE_VV(FMACC, FMADD)
4523 CASE_VFMA_CHANGE_OPCODE_VV(FMSAC, FMSUB)
4524 CASE_VFMA_CHANGE_OPCODE_VV(FNMACC, FNMADD)
4525 CASE_VFMA_CHANGE_OPCODE_VV(FNMSAC, FNMSUB)
4526 CASE_VMA_CHANGE_OPCODE_LMULS(MACC, MADD, VX)
4527 CASE_VMA_CHANGE_OPCODE_LMULS(MADD, MACC, VX)
4528 CASE_VMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VX)
4529 CASE_VMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VX)
4530 CASE_VMA_CHANGE_OPCODE_LMULS(MACC, MADD, VV)
4531 CASE_VMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VV)
4532 }
4533
4534 auto &WorkingMI = cloneIfNew(MI);
4535 WorkingMI.setDesc(get(Opcode: Opc));
4536 return TargetInstrInfo::commuteInstructionImpl(MI&: WorkingMI, /*NewMI=*/false,
4537 OpIdx1, OpIdx2);
4538 }
4539 case CASE_VFMA_OPCODE_VV(FMADD):
4540 case CASE_VFMA_OPCODE_VV(FMSUB):
4541 case CASE_VFMA_OPCODE_VV(FNMADD):
4542 case CASE_VFMA_OPCODE_VV(FNMSUB):
4543 case CASE_VMA_OPCODE_LMULS(MADD, VV):
4544 case CASE_VMA_OPCODE_LMULS(NMSUB, VV): {
4545 assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");
4546 // If one of the operands, is the addend we need to change opcode.
4547 // Otherwise we're just swapping 2 of the multiplicands.
4548 if (OpIdx1 == 3 || OpIdx2 == 3) {
4549 unsigned Opc;
4550 switch (MI.getOpcode()) {
4551 default:
4552 llvm_unreachable("Unexpected opcode");
4553 CASE_VFMA_CHANGE_OPCODE_VV(FMADD, FMACC)
4554 CASE_VFMA_CHANGE_OPCODE_VV(FMSUB, FMSAC)
4555 CASE_VFMA_CHANGE_OPCODE_VV(FNMADD, FNMACC)
4556 CASE_VFMA_CHANGE_OPCODE_VV(FNMSUB, FNMSAC)
4557 CASE_VMA_CHANGE_OPCODE_LMULS(MADD, MACC, VV)
4558 CASE_VMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VV)
4559 }
4560
4561 auto &WorkingMI = cloneIfNew(MI);
4562 WorkingMI.setDesc(get(Opcode: Opc));
4563 return TargetInstrInfo::commuteInstructionImpl(MI&: WorkingMI, /*NewMI=*/false,
4564 OpIdx1, OpIdx2);
4565 }
4566 // Let the default code handle it.
4567 break;
4568 }
4569 }
4570
4571 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
4572}
4573
4574#undef CASE_VMA_CHANGE_OPCODE_COMMON
4575#undef CASE_VMA_CHANGE_OPCODE_LMULS
4576#undef CASE_VFMA_CHANGE_OPCODE_COMMON
4577#undef CASE_VFMA_CHANGE_OPCODE_LMULS_M1
4578#undef CASE_VFMA_CHANGE_OPCODE_LMULS_MF2
4579#undef CASE_VFMA_CHANGE_OPCODE_LMULS_MF4
4580#undef CASE_VFMA_CHANGE_OPCODE_VV
4581#undef CASE_VFMA_CHANGE_OPCODE_SPLATS
4582
4583#undef CASE_RVV_OPCODE_UNMASK_LMUL
4584#undef CASE_RVV_OPCODE_MASK_LMUL
4585#undef CASE_RVV_OPCODE_LMUL
4586#undef CASE_RVV_OPCODE_UNMASK_WIDEN
4587#undef CASE_RVV_OPCODE_UNMASK
4588#undef CASE_RVV_OPCODE_MASK_WIDEN
4589#undef CASE_RVV_OPCODE_MASK
4590#undef CASE_RVV_OPCODE_WIDEN
4591#undef CASE_RVV_OPCODE
4592
4593#undef CASE_VMA_OPCODE_COMMON
4594#undef CASE_VMA_OPCODE_LMULS
4595#undef CASE_VFMA_OPCODE_COMMON
4596#undef CASE_VFMA_OPCODE_LMULS_M1
4597#undef CASE_VFMA_OPCODE_LMULS_MF2
4598#undef CASE_VFMA_OPCODE_LMULS_MF4
4599#undef CASE_VFMA_OPCODE_VV
4600#undef CASE_VFMA_SPLATS
4601
4602bool RISCVInstrInfo::simplifyInstruction(MachineInstr &MI) const {
4603 switch (MI.getOpcode()) {
4604 default:
4605 break;
4606 case RISCV::ADD:
4607 case RISCV::OR:
4608 case RISCV::XOR:
4609 // Normalize (so we hit the next if clause).
4610 // add/[x]or rd, zero, rs => add/[x]or rd, rs, zero
4611 if (MI.getOperand(i: 1).getReg() == RISCV::X0)
4612 commuteInstruction(MI);
4613 // add/[x]or rd, rs, zero => addi rd, rs, 0
4614 if (MI.getOperand(i: 2).getReg() == RISCV::X0) {
4615 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4616 MI.setDesc(get(Opcode: RISCV::ADDI));
4617 return true;
4618 }
4619 // xor rd, rs, rs => addi rd, zero, 0
4620 if (MI.getOpcode() == RISCV::XOR &&
4621 MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg()) {
4622 MI.getOperand(i: 1).setReg(RISCV::X0);
4623 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4624 MI.setDesc(get(Opcode: RISCV::ADDI));
4625 return true;
4626 }
4627 break;
4628 case RISCV::ORI:
4629 case RISCV::XORI:
4630 // [x]ori rd, zero, N => addi rd, zero, N
4631 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4632 MI.setDesc(get(Opcode: RISCV::ADDI));
4633 return true;
4634 }
4635 break;
4636 case RISCV::SUB:
4637 // sub rd, rs, zero => addi rd, rs, 0
4638 if (MI.getOperand(i: 2).getReg() == RISCV::X0) {
4639 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4640 MI.setDesc(get(Opcode: RISCV::ADDI));
4641 return true;
4642 }
4643 break;
4644 case RISCV::SUBW:
4645 // subw rd, rs, zero => addiw rd, rs, 0
4646 if (MI.getOperand(i: 2).getReg() == RISCV::X0) {
4647 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4648 MI.setDesc(get(Opcode: RISCV::ADDIW));
4649 return true;
4650 }
4651 break;
4652 case RISCV::ADDW:
4653 // Normalize (so we hit the next if clause).
4654 // addw rd, zero, rs => addw rd, rs, zero
4655 if (MI.getOperand(i: 1).getReg() == RISCV::X0)
4656 commuteInstruction(MI);
4657 // addw rd, rs, zero => addiw rd, rs, 0
4658 if (MI.getOperand(i: 2).getReg() == RISCV::X0) {
4659 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4660 MI.setDesc(get(Opcode: RISCV::ADDIW));
4661 return true;
4662 }
4663 break;
4664 case RISCV::SH1ADD:
4665 case RISCV::SH1ADD_UW:
4666 case RISCV::SH2ADD:
4667 case RISCV::SH2ADD_UW:
4668 case RISCV::SH3ADD:
4669 case RISCV::SH3ADD_UW:
4670 // shNadd[.uw] rd, zero, rs => addi rd, rs, 0
4671 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4672 MI.removeOperand(OpNo: 1);
4673 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
4674 MI.setDesc(get(Opcode: RISCV::ADDI));
4675 return true;
4676 }
4677 // shNadd[.uw] rd, rs, zero => slli[.uw] rd, rs, N
4678 if (MI.getOperand(i: 2).getReg() == RISCV::X0) {
4679 MI.removeOperand(OpNo: 2);
4680 unsigned Opc = MI.getOpcode();
4681 if (Opc == RISCV::SH1ADD_UW || Opc == RISCV::SH2ADD_UW ||
4682 Opc == RISCV::SH3ADD_UW) {
4683 MI.addOperand(Op: MachineOperand::CreateImm(Val: getSHXADDUWShiftAmount(Opc)));
4684 MI.setDesc(get(Opcode: RISCV::SLLI_UW));
4685 return true;
4686 }
4687 MI.addOperand(Op: MachineOperand::CreateImm(Val: getSHXADDShiftAmount(Opc)));
4688 MI.setDesc(get(Opcode: RISCV::SLLI));
4689 return true;
4690 }
4691 break;
4692 case RISCV::AND:
4693 case RISCV::MUL:
4694 case RISCV::MULH:
4695 case RISCV::MULHSU:
4696 case RISCV::MULHU:
4697 case RISCV::MULW:
4698 // and rd, zero, rs => addi rd, zero, 0
4699 // mul* rd, zero, rs => addi rd, zero, 0
4700 // and rd, rs, zero => addi rd, zero, 0
4701 // mul* rd, rs, zero => addi rd, zero, 0
4702 if (MI.getOperand(i: 1).getReg() == RISCV::X0 ||
4703 MI.getOperand(i: 2).getReg() == RISCV::X0) {
4704 MI.getOperand(i: 1).setReg(RISCV::X0);
4705 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4706 MI.setDesc(get(Opcode: RISCV::ADDI));
4707 return true;
4708 }
4709 break;
4710 case RISCV::ANDI:
4711 // andi rd, zero, C => addi rd, zero, 0
4712 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4713 MI.getOperand(i: 2).setImm(0);
4714 MI.setDesc(get(Opcode: RISCV::ADDI));
4715 return true;
4716 }
4717 break;
4718 case RISCV::SLL:
4719 case RISCV::SRL:
4720 case RISCV::SRA:
4721 // shift rd, zero, rs => addi rd, zero, 0
4722 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4723 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4724 MI.setDesc(get(Opcode: RISCV::ADDI));
4725 return true;
4726 }
4727 // shift rd, rs, zero => addi rd, rs, 0
4728 if (MI.getOperand(i: 2).getReg() == RISCV::X0) {
4729 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4730 MI.setDesc(get(Opcode: RISCV::ADDI));
4731 return true;
4732 }
4733 break;
4734 case RISCV::SLLW:
4735 case RISCV::SRLW:
4736 case RISCV::SRAW:
4737 // shiftw rd, zero, rs => addi rd, zero, 0
4738 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4739 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4740 MI.setDesc(get(Opcode: RISCV::ADDI));
4741 return true;
4742 }
4743 break;
4744 case RISCV::SLLI:
4745 case RISCV::SRLI:
4746 case RISCV::SRAI:
4747 case RISCV::SLLIW:
4748 case RISCV::SRLIW:
4749 case RISCV::SRAIW:
4750 case RISCV::SLLI_UW:
4751 // shiftimm rd, zero, N => addi rd, zero, 0
4752 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4753 MI.getOperand(i: 2).setImm(0);
4754 MI.setDesc(get(Opcode: RISCV::ADDI));
4755 return true;
4756 }
4757 break;
4758 case RISCV::SLTU:
4759 case RISCV::ADD_UW:
4760 // sltu rd, zero, zero => addi rd, zero, 0
4761 // add.uw rd, zero, zero => addi rd, zero, 0
4762 if (MI.getOperand(i: 1).getReg() == RISCV::X0 &&
4763 MI.getOperand(i: 2).getReg() == RISCV::X0) {
4764 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4765 MI.setDesc(get(Opcode: RISCV::ADDI));
4766 return true;
4767 }
4768 // add.uw rd, zero, rs => addi rd, rs, 0
4769 if (MI.getOpcode() == RISCV::ADD_UW &&
4770 MI.getOperand(i: 1).getReg() == RISCV::X0) {
4771 MI.removeOperand(OpNo: 1);
4772 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
4773 MI.setDesc(get(Opcode: RISCV::ADDI));
4774 }
4775 break;
4776 case RISCV::SLTIU:
4777 // sltiu rd, zero, NZC => addi rd, zero, 1
4778 // sltiu rd, zero, 0 => addi rd, zero, 0
4779 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4780 MI.getOperand(i: 2).setImm(MI.getOperand(i: 2).getImm() != 0);
4781 MI.setDesc(get(Opcode: RISCV::ADDI));
4782 return true;
4783 }
4784 break;
4785 case RISCV::SEXT_H:
4786 case RISCV::SEXT_B:
4787 case RISCV::ZEXT_H_RV32:
4788 case RISCV::ZEXT_H_RV64:
4789 // sext.[hb] rd, zero => addi rd, zero, 0
4790 // zext.h rd, zero => addi rd, zero, 0
4791 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4792 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
4793 MI.setDesc(get(Opcode: RISCV::ADDI));
4794 return true;
4795 }
4796 break;
4797 case RISCV::MIN:
4798 case RISCV::MINU:
4799 case RISCV::MAX:
4800 case RISCV::MAXU:
4801 // min|max rd, rs, rs => addi rd, rs, 0
4802 if (MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg()) {
4803 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4804 MI.setDesc(get(Opcode: RISCV::ADDI));
4805 return true;
4806 }
4807 break;
4808 case RISCV::BEQ:
4809 case RISCV::BNE:
4810 // b{eq,ne} zero, rs, imm => b{eq,ne} rs, zero, imm
4811 if (MI.getOperand(i: 0).getReg() == RISCV::X0) {
4812 MachineOperand MO0 = MI.getOperand(i: 0);
4813 MI.removeOperand(OpNo: 0);
4814 MI.insert(InsertBefore: MI.operands_begin() + 1, Ops: {MO0});
4815 }
4816 break;
4817 case RISCV::BLTU:
4818 // bltu zero, rs, imm => bne rs, zero, imm
4819 if (MI.getOperand(i: 0).getReg() == RISCV::X0) {
4820 MachineOperand MO0 = MI.getOperand(i: 0);
4821 MI.removeOperand(OpNo: 0);
4822 MI.insert(InsertBefore: MI.operands_begin() + 1, Ops: {MO0});
4823 MI.setDesc(get(Opcode: RISCV::BNE));
4824 }
4825 break;
4826 case RISCV::BGEU:
4827 // bgeu zero, rs, imm => beq rs, zero, imm
4828 if (MI.getOperand(i: 0).getReg() == RISCV::X0) {
4829 MachineOperand MO0 = MI.getOperand(i: 0);
4830 MI.removeOperand(OpNo: 0);
4831 MI.insert(InsertBefore: MI.operands_begin() + 1, Ops: {MO0});
4832 MI.setDesc(get(Opcode: RISCV::BEQ));
4833 }
4834 break;
4835 }
4836 return false;
4837}
4838
4839// clang-format off
4840#define CASE_WIDEOP_OPCODE_COMMON(OP, LMUL) \
4841 RISCV::PseudoV##OP##_##LMUL##_TIED
4842
4843#define CASE_WIDEOP_OPCODE_LMULS(OP) \
4844 CASE_WIDEOP_OPCODE_COMMON(OP, MF8): \
4845 case CASE_WIDEOP_OPCODE_COMMON(OP, MF4): \
4846 case CASE_WIDEOP_OPCODE_COMMON(OP, MF2): \
4847 case CASE_WIDEOP_OPCODE_COMMON(OP, M1): \
4848 case CASE_WIDEOP_OPCODE_COMMON(OP, M2): \
4849 case CASE_WIDEOP_OPCODE_COMMON(OP, M4)
4850
4851#define CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL) \
4852 case RISCV::PseudoV##OP##_##LMUL##_TIED: \
4853 NewOpc = RISCV::PseudoV##OP##_##LMUL; \
4854 break;
4855
4856#define CASE_WIDEOP_CHANGE_OPCODE_LMULS(OP) \
4857 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF8) \
4858 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4) \
4859 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2) \
4860 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1) \
4861 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2) \
4862 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4)
4863
4864// FP Widening Ops may by SEW aware. Create SEW aware cases for these cases.
4865#define CASE_FP_WIDEOP_OPCODE_COMMON(OP, LMUL, SEW) \
4866 RISCV::PseudoV##OP##_##LMUL##_##SEW##_TIED
4867
4868#define CASE_FP_WIDEOP_OPCODE_LMULS(OP) \
4869 CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF4, E16): \
4870 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E16): \
4871 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E32): \
4872 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E16): \
4873 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E32): \
4874 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E16): \
4875 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E32): \
4876 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E16): \
4877 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E32) \
4878
4879#define CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL, SEW) \
4880 case RISCV::PseudoV##OP##_##LMUL##_##SEW##_TIED: \
4881 NewOpc = RISCV::PseudoV##OP##_##LMUL##_##SEW; \
4882 break;
4883
4884#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(OP) \
4885 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4, E16) \
4886 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E16) \
4887 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E32) \
4888 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E16) \
4889 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E32) \
4890 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E16) \
4891 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E32) \
4892 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E16) \
4893 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E32) \
4894
4895#define CASE_FP_WIDEOP_OPCODE_LMULS_ALT(OP) \
4896 CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF4, E16): \
4897 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E16): \
4898 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E16): \
4899 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E16): \
4900 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E16)
4901
4902#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_ALT(OP) \
4903 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4, E16) \
4904 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E16) \
4905 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E16) \
4906 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E16) \
4907 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E16)
4908// clang-format on
4909
4910MachineInstr *RISCVInstrInfo::convertToThreeAddress(MachineInstr &MI,
4911 LiveVariables *LV,
4912 LiveIntervals *LIS) const {
4913 MachineInstrBuilder MIB;
4914 switch (MI.getOpcode()) {
4915 default:
4916 return nullptr;
4917 case CASE_FP_WIDEOP_OPCODE_LMULS_ALT(FWADD_ALT_WV):
4918 case CASE_FP_WIDEOP_OPCODE_LMULS_ALT(FWSUB_ALT_WV):
4919 case CASE_FP_WIDEOP_OPCODE_LMULS(FWADD_WV):
4920 case CASE_FP_WIDEOP_OPCODE_LMULS(FWSUB_WV): {
4921 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags) &&
4922 MI.getNumExplicitOperands() == 7 &&
4923 "Expect 7 explicit operands rd, rs2, rs1, rm, vl, sew, policy");
4924 // If the tail policy is undisturbed we can't convert.
4925 if ((MI.getOperand(i: RISCVII::getVecPolicyOpNum(Desc: MI.getDesc())).getImm() &
4926 1) == 0)
4927 return nullptr;
4928 // clang-format off
4929 unsigned NewOpc;
4930 switch (MI.getOpcode()) {
4931 default:
4932 llvm_unreachable("Unexpected opcode");
4933 CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(FWADD_WV)
4934 CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(FWSUB_WV)
4935 CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_ALT(FWADD_ALT_WV)
4936 CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_ALT(FWSUB_ALT_WV)
4937 }
4938 // clang-format on
4939
4940 MachineBasicBlock &MBB = *MI.getParent();
4941 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4942 .add(MO: MI.getOperand(i: 0))
4943 .addReg(RegNo: MI.getOperand(i: 0).getReg(), Flags: RegState::Undef)
4944 .add(MO: MI.getOperand(i: 1))
4945 .add(MO: MI.getOperand(i: 2))
4946 .add(MO: MI.getOperand(i: 3))
4947 .add(MO: MI.getOperand(i: 4))
4948 .add(MO: MI.getOperand(i: 5))
4949 .add(MO: MI.getOperand(i: 6));
4950 break;
4951 }
4952 case CASE_WIDEOP_OPCODE_LMULS(WADD_WV):
4953 case CASE_WIDEOP_OPCODE_LMULS(WADDU_WV):
4954 case CASE_WIDEOP_OPCODE_LMULS(WSUB_WV):
4955 case CASE_WIDEOP_OPCODE_LMULS(WSUBU_WV): {
4956 // If the tail policy is undisturbed we can't convert.
4957 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags) &&
4958 MI.getNumExplicitOperands() == 6);
4959 if ((MI.getOperand(i: RISCVII::getVecPolicyOpNum(Desc: MI.getDesc())).getImm() &
4960 1) == 0)
4961 return nullptr;
4962
4963 // clang-format off
4964 unsigned NewOpc;
4965 switch (MI.getOpcode()) {
4966 default:
4967 llvm_unreachable("Unexpected opcode");
4968 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WADD_WV)
4969 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WADDU_WV)
4970 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WSUB_WV)
4971 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WSUBU_WV)
4972 }
4973 // clang-format on
4974
4975 MachineBasicBlock &MBB = *MI.getParent();
4976 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4977 .add(MO: MI.getOperand(i: 0))
4978 .addReg(RegNo: MI.getOperand(i: 0).getReg(), Flags: RegState::Undef)
4979 .add(MO: MI.getOperand(i: 1))
4980 .add(MO: MI.getOperand(i: 2))
4981 .add(MO: MI.getOperand(i: 3))
4982 .add(MO: MI.getOperand(i: 4))
4983 .add(MO: MI.getOperand(i: 5));
4984 break;
4985 }
4986 }
4987 MIB.copyImplicitOps(OtherMI: MI);
4988
4989 if (LV) {
4990 unsigned NumOps = MI.getNumOperands();
4991 for (unsigned I = 1; I < NumOps; ++I) {
4992 MachineOperand &Op = MI.getOperand(i: I);
4993 if (Op.isReg() && Op.isKill())
4994 LV->replaceKillInstruction(Reg: Op.getReg(), OldMI&: MI, NewMI&: *MIB);
4995 }
4996 }
4997
4998 if (LIS) {
4999 SlotIndex Idx = LIS->ReplaceMachineInstrInMaps(MI, NewMI&: *MIB);
5000
5001 if (MI.getOperand(i: 0).isEarlyClobber()) {
5002 // Use operand 1 was tied to early-clobber def operand 0, so its live
5003 // interval could have ended at an early-clobber slot. Now they are not
5004 // tied we need to update it to the normal register slot.
5005 LiveInterval &LI = LIS->getInterval(Reg: MI.getOperand(i: 1).getReg());
5006 LiveRange::Segment *S = LI.getSegmentContaining(Idx);
5007 if (S->end == Idx.getRegSlot(EC: true))
5008 S->end = Idx.getRegSlot();
5009 }
5010 }
5011
5012 return MIB;
5013}
5014
5015#undef CASE_WIDEOP_OPCODE_COMMON
5016#undef CASE_WIDEOP_OPCODE_LMULS
5017#undef CASE_WIDEOP_CHANGE_OPCODE_COMMON
5018#undef CASE_WIDEOP_CHANGE_OPCODE_LMULS
5019#undef CASE_FP_WIDEOP_OPCODE_COMMON
5020#undef CASE_FP_WIDEOP_OPCODE_LMULS
5021#undef CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON
5022#undef CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS
5023
5024void RISCVInstrInfo::mulImm(MachineFunction &MF, MachineBasicBlock &MBB,
5025 MachineBasicBlock::iterator II, const DebugLoc &DL,
5026 Register DestReg, uint32_t Amount,
5027 MachineInstr::MIFlag Flag) const {
5028 MachineRegisterInfo &MRI = MF.getRegInfo();
5029 if (llvm::has_single_bit(Value: Amount)) {
5030 uint32_t ShiftAmount = Log2_32(Value: Amount);
5031 if (ShiftAmount == 0)
5032 return;
5033 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::SLLI), DestReg)
5034 .addReg(RegNo: DestReg, Flags: RegState::Kill)
5035 .addImm(Val: ShiftAmount)
5036 .setMIFlag(Flag);
5037 } else if (int ShXAmount, ShiftAmount;
5038 STI.hasShlAdd(ShAmt: 3) &&
5039 (ShXAmount = isShifted359(Value: Amount, Shift&: ShiftAmount)) != 0) {
5040 // We can use Zba SHXADD+SLLI instructions for multiply in some cases.
5041 unsigned Opc;
5042 switch (ShXAmount) {
5043 case 1:
5044 Opc = RISCV::SH1ADD;
5045 break;
5046 case 2:
5047 Opc = RISCV::SH2ADD;
5048 break;
5049 case 3:
5050 Opc = RISCV::SH3ADD;
5051 break;
5052 default:
5053 llvm_unreachable("unexpected result of isShifted359");
5054 }
5055 if (ShiftAmount)
5056 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::SLLI), DestReg)
5057 .addReg(RegNo: DestReg, Flags: RegState::Kill)
5058 .addImm(Val: ShiftAmount)
5059 .setMIFlag(Flag);
5060 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: Opc), DestReg)
5061 .addReg(RegNo: DestReg, Flags: RegState::Kill)
5062 .addReg(RegNo: DestReg)
5063 .setMIFlag(Flag);
5064 } else if (llvm::has_single_bit(Value: Amount - 1)) {
5065 Register ScaledRegister = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
5066 uint32_t ShiftAmount = Log2_32(Value: Amount - 1);
5067 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::SLLI), DestReg: ScaledRegister)
5068 .addReg(RegNo: DestReg)
5069 .addImm(Val: ShiftAmount)
5070 .setMIFlag(Flag);
5071 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::ADD), DestReg)
5072 .addReg(RegNo: ScaledRegister, Flags: RegState::Kill)
5073 .addReg(RegNo: DestReg, Flags: RegState::Kill)
5074 .setMIFlag(Flag);
5075 } else if (llvm::has_single_bit(Value: Amount + 1)) {
5076 Register ScaledRegister = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
5077 uint32_t ShiftAmount = Log2_32(Value: Amount + 1);
5078 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::SLLI), DestReg: ScaledRegister)
5079 .addReg(RegNo: DestReg)
5080 .addImm(Val: ShiftAmount)
5081 .setMIFlag(Flag);
5082 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::SUB), DestReg)
5083 .addReg(RegNo: ScaledRegister, Flags: RegState::Kill)
5084 .addReg(RegNo: DestReg, Flags: RegState::Kill)
5085 .setMIFlag(Flag);
5086 } else if (STI.hasStdExtZmmul()) {
5087 Register N = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
5088 movImm(MBB, MBBI: II, DL, DstReg: N, Val: Amount, Flag);
5089 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::MUL), DestReg)
5090 .addReg(RegNo: DestReg, Flags: RegState::Kill)
5091 .addReg(RegNo: N, Flags: RegState::Kill)
5092 .setMIFlag(Flag);
5093 } else {
5094 Register Acc;
5095 uint32_t PrevShiftAmount = 0;
5096 for (uint32_t ShiftAmount = 0; Amount >> ShiftAmount; ShiftAmount++) {
5097 if (Amount & (1U << ShiftAmount)) {
5098 if (ShiftAmount)
5099 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::SLLI), DestReg)
5100 .addReg(RegNo: DestReg, Flags: RegState::Kill)
5101 .addImm(Val: ShiftAmount - PrevShiftAmount)
5102 .setMIFlag(Flag);
5103 if (Amount >> (ShiftAmount + 1)) {
5104 // If we don't have an accmulator yet, create it and copy DestReg.
5105 if (!Acc) {
5106 Acc = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
5107 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: TargetOpcode::COPY), DestReg: Acc)
5108 .addReg(RegNo: DestReg)
5109 .setMIFlag(Flag);
5110 } else {
5111 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::ADD), DestReg: Acc)
5112 .addReg(RegNo: Acc, Flags: RegState::Kill)
5113 .addReg(RegNo: DestReg)
5114 .setMIFlag(Flag);
5115 }
5116 }
5117 PrevShiftAmount = ShiftAmount;
5118 }
5119 }
5120 assert(Acc && "Expected valid accumulator");
5121 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::ADD), DestReg)
5122 .addReg(RegNo: DestReg, Flags: RegState::Kill)
5123 .addReg(RegNo: Acc, Flags: RegState::Kill)
5124 .setMIFlag(Flag);
5125 }
5126}
5127
5128ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
5129RISCVInstrInfo::getSerializableMachineMemOperandTargetFlags() const {
5130 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
5131 {{MONontemporalBit0, "riscv-nontemporal-domain-bit-0"},
5132 {MONontemporalBit1, "riscv-nontemporal-domain-bit-1"}};
5133 return ArrayRef(TargetFlags);
5134}
5135
5136unsigned RISCVInstrInfo::getTailDuplicateSize(CodeGenOptLevel OptLevel) const {
5137 return OptLevel >= CodeGenOptLevel::Aggressive
5138 ? STI.getTailDupAggressiveThreshold()
5139 : 2;
5140}
5141
5142bool RISCV::isRVVSpill(const MachineInstr &MI) {
5143 // RVV lacks any support for immediate addressing for stack addresses, so be
5144 // conservative.
5145 unsigned Opcode = MI.getOpcode();
5146 if (!RISCVVPseudosTable::getPseudoInfo(Pseudo: Opcode) &&
5147 !getLMULForRVVWholeLoadStore(Opcode) && !isRVVSpillForZvlsseg(Opcode))
5148 return false;
5149 return true;
5150}
5151
5152/// Return true if \p MI is a copy that will be lowered to one or more vmvNr.vs.
5153bool RISCV::isVectorCopy(const TargetRegisterInfo *TRI,
5154 const MachineInstr &MI) {
5155 return MI.isCopy() && MI.getOperand(i: 0).getReg().isPhysical() &&
5156 RISCVRegisterInfo::isRVVRegClass(
5157 RC: TRI->getMinimalPhysRegClass(Reg: MI.getOperand(i: 0).getReg()));
5158}
5159
5160std::optional<std::pair<unsigned, unsigned>>
5161RISCV::isRVVSpillForZvlsseg(unsigned Opcode) {
5162 switch (Opcode) {
5163 default:
5164 return std::nullopt;
5165 case RISCV::PseudoVSPILL2_M1:
5166 case RISCV::PseudoVRELOAD2_M1:
5167 return std::make_pair(x: 2u, y: 1u);
5168 case RISCV::PseudoVSPILL2_M2:
5169 case RISCV::PseudoVRELOAD2_M2:
5170 return std::make_pair(x: 2u, y: 2u);
5171 case RISCV::PseudoVSPILL2_M4:
5172 case RISCV::PseudoVRELOAD2_M4:
5173 return std::make_pair(x: 2u, y: 4u);
5174 case RISCV::PseudoVSPILL3_M1:
5175 case RISCV::PseudoVRELOAD3_M1:
5176 return std::make_pair(x: 3u, y: 1u);
5177 case RISCV::PseudoVSPILL3_M2:
5178 case RISCV::PseudoVRELOAD3_M2:
5179 return std::make_pair(x: 3u, y: 2u);
5180 case RISCV::PseudoVSPILL4_M1:
5181 case RISCV::PseudoVRELOAD4_M1:
5182 return std::make_pair(x: 4u, y: 1u);
5183 case RISCV::PseudoVSPILL4_M2:
5184 case RISCV::PseudoVRELOAD4_M2:
5185 return std::make_pair(x: 4u, y: 2u);
5186 case RISCV::PseudoVSPILL5_M1:
5187 case RISCV::PseudoVRELOAD5_M1:
5188 return std::make_pair(x: 5u, y: 1u);
5189 case RISCV::PseudoVSPILL6_M1:
5190 case RISCV::PseudoVRELOAD6_M1:
5191 return std::make_pair(x: 6u, y: 1u);
5192 case RISCV::PseudoVSPILL7_M1:
5193 case RISCV::PseudoVRELOAD7_M1:
5194 return std::make_pair(x: 7u, y: 1u);
5195 case RISCV::PseudoVSPILL8_M1:
5196 case RISCV::PseudoVRELOAD8_M1:
5197 return std::make_pair(x: 8u, y: 1u);
5198 }
5199}
5200
5201bool RISCV::hasEqualFRM(const MachineInstr &MI1, const MachineInstr &MI2) {
5202 int16_t MI1FrmOpIdx =
5203 RISCV::getNamedOperandIdx(Opcode: MI1.getOpcode(), Name: RISCV::OpName::frm);
5204 int16_t MI2FrmOpIdx =
5205 RISCV::getNamedOperandIdx(Opcode: MI2.getOpcode(), Name: RISCV::OpName::frm);
5206 if (MI1FrmOpIdx < 0 || MI2FrmOpIdx < 0)
5207 return false;
5208 MachineOperand FrmOp1 = MI1.getOperand(i: MI1FrmOpIdx);
5209 MachineOperand FrmOp2 = MI2.getOperand(i: MI2FrmOpIdx);
5210 return FrmOp1.getImm() == FrmOp2.getImm();
5211}
5212
5213std::optional<unsigned>
5214RISCV::getVectorLowDemandedScalarBits(unsigned Opcode, unsigned Log2SEW) {
5215 switch (Opcode) {
5216 default:
5217 return std::nullopt;
5218
5219 // 11.6. Vector Single-Width Shift Instructions
5220 case RISCV::VSLL_VX:
5221 case RISCV::VSRL_VX:
5222 case RISCV::VSRA_VX:
5223 // 12.4. Vector Single-Width Scaling Shift Instructions
5224 case RISCV::VSSRL_VX:
5225 case RISCV::VSSRA_VX:
5226 // Zvbb
5227 case RISCV::VROL_VX:
5228 case RISCV::VROR_VX:
5229 // Only the low lg2(SEW) bits of the shift-amount value are used.
5230 return Log2SEW;
5231
5232 // 11.7 Vector Narrowing Integer Right Shift Instructions
5233 case RISCV::VNSRL_WX:
5234 case RISCV::VNSRA_WX:
5235 // 12.5. Vector Narrowing Fixed-Point Clip Instructions
5236 case RISCV::VNCLIPU_WX:
5237 case RISCV::VNCLIP_WX:
5238 // Zvbb
5239 case RISCV::VWSLL_VX:
5240 // Only the low lg2(2*SEW) bits of the shift-amount value are used.
5241 return Log2SEW + 1;
5242
5243 // 11.1. Vector Single-Width Integer Add and Subtract
5244 case RISCV::VADD_VX:
5245 case RISCV::VSUB_VX:
5246 case RISCV::VRSUB_VX:
5247 // 11.2. Vector Widening Integer Add/Subtract
5248 case RISCV::VWADDU_VX:
5249 case RISCV::VWSUBU_VX:
5250 case RISCV::VWADD_VX:
5251 case RISCV::VWSUB_VX:
5252 case RISCV::VWADDU_WX:
5253 case RISCV::VWSUBU_WX:
5254 case RISCV::VWADD_WX:
5255 case RISCV::VWSUB_WX:
5256 // 11.4. Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions
5257 case RISCV::VADC_VXM:
5258 case RISCV::VADC_VIM:
5259 case RISCV::VMADC_VXM:
5260 case RISCV::VMADC_VIM:
5261 case RISCV::VMADC_VX:
5262 case RISCV::VSBC_VXM:
5263 case RISCV::VMSBC_VXM:
5264 case RISCV::VMSBC_VX:
5265 // 11.5 Vector Bitwise Logical Instructions
5266 case RISCV::VAND_VX:
5267 case RISCV::VOR_VX:
5268 case RISCV::VXOR_VX:
5269 // 11.8. Vector Integer Compare Instructions
5270 case RISCV::VMSEQ_VX:
5271 case RISCV::VMSNE_VX:
5272 case RISCV::VMSLTU_VX:
5273 case RISCV::VMSLT_VX:
5274 case RISCV::VMSLEU_VX:
5275 case RISCV::VMSLE_VX:
5276 case RISCV::VMSGTU_VX:
5277 case RISCV::VMSGT_VX:
5278 // 11.9. Vector Integer Min/Max Instructions
5279 case RISCV::VMINU_VX:
5280 case RISCV::VMIN_VX:
5281 case RISCV::VMAXU_VX:
5282 case RISCV::VMAX_VX:
5283 // 11.10. Vector Single-Width Integer Multiply Instructions
5284 case RISCV::VMUL_VX:
5285 case RISCV::VMULH_VX:
5286 case RISCV::VMULHU_VX:
5287 case RISCV::VMULHSU_VX:
5288 // 11.11. Vector Integer Divide Instructions
5289 case RISCV::VDIVU_VX:
5290 case RISCV::VDIV_VX:
5291 case RISCV::VREMU_VX:
5292 case RISCV::VREM_VX:
5293 // 11.12. Vector Widening Integer Multiply Instructions
5294 case RISCV::VWMUL_VX:
5295 case RISCV::VWMULU_VX:
5296 case RISCV::VWMULSU_VX:
5297 // 11.13. Vector Single-Width Integer Multiply-Add Instructions
5298 case RISCV::VMACC_VX:
5299 case RISCV::VNMSAC_VX:
5300 case RISCV::VMADD_VX:
5301 case RISCV::VNMSUB_VX:
5302 // 11.14. Vector Widening Integer Multiply-Add Instructions
5303 case RISCV::VWMACCU_VX:
5304 case RISCV::VWMACC_VX:
5305 case RISCV::VWMACCSU_VX:
5306 case RISCV::VWMACCUS_VX:
5307 // 11.15. Vector Integer Merge Instructions
5308 case RISCV::VMERGE_VXM:
5309 // 11.16. Vector Integer Move Instructions
5310 case RISCV::VMV_V_X:
5311 // 12.1. Vector Single-Width Saturating Add and Subtract
5312 case RISCV::VSADDU_VX:
5313 case RISCV::VSADD_VX:
5314 case RISCV::VSSUBU_VX:
5315 case RISCV::VSSUB_VX:
5316 // 12.2. Vector Single-Width Averaging Add and Subtract
5317 case RISCV::VAADDU_VX:
5318 case RISCV::VAADD_VX:
5319 case RISCV::VASUBU_VX:
5320 case RISCV::VASUB_VX:
5321 // 12.3. Vector Single-Width Fractional Multiply with Rounding and Saturation
5322 case RISCV::VSMUL_VX:
5323 // 16.1. Integer Scalar Move Instructions
5324 case RISCV::VMV_S_X:
5325 // Zvbb
5326 case RISCV::VANDN_VX:
5327 return 1U << Log2SEW;
5328 }
5329}
5330
5331unsigned RISCV::getRVVMCOpcode(unsigned RVVPseudoOpcode) {
5332 const RISCVVPseudosTable::PseudoInfo *RVV =
5333 RISCVVPseudosTable::getPseudoInfo(Pseudo: RVVPseudoOpcode);
5334 if (!RVV)
5335 return 0;
5336 return RVV->BaseInstr;
5337}
5338
5339unsigned RISCV::getDestLog2EEW(const MCInstrDesc &Desc, unsigned Log2SEW) {
5340 unsigned DestEEW =
5341 (Desc.TSFlags & RISCVII::DestEEWMask) >> RISCVII::DestEEWShift;
5342 // EEW = 1
5343 if (DestEEW == 0)
5344 return 0;
5345 // EEW = SEW * n
5346 unsigned Scaled = Log2SEW + (DestEEW - 1);
5347 assert(Scaled >= 3 && Scaled <= 6);
5348 return Scaled;
5349}
5350
5351static std::optional<int64_t> getEffectiveImm(const MachineRegisterInfo &MRI,
5352 const MachineOperand &MO) {
5353 assert(MO.isImm() || MO.getReg().isVirtual());
5354 if (MO.isImm())
5355 return MO.getImm();
5356 const MachineInstr *Def = MRI.getVRegDef(Reg: MO.getReg());
5357 int64_t Imm;
5358 if (isLoadImm(MI: Def, Imm))
5359 return Imm;
5360 return std::nullopt;
5361}
5362
5363/// Given two VL operands, do we know that LHS <= RHS? Must be used in SSA form.
5364bool RISCV::isVLKnownLE(const MachineRegisterInfo &MRI,
5365 const MachineOperand &LHS, const MachineOperand &RHS) {
5366 assert((LHS.isImm() || MRI.isSSA()) && (RHS.isImm() || MRI.isSSA()));
5367 if (LHS.isReg() && RHS.isReg() && LHS.getReg().isVirtual() &&
5368 LHS.getReg() == RHS.getReg())
5369 return true;
5370 if (RHS.isImm() && RHS.getImm() == RISCV::VLMaxSentinel)
5371 return true;
5372 if (LHS.isImm() && LHS.getImm() == 0)
5373 return true;
5374 if (LHS.isImm() && LHS.getImm() == RISCV::VLMaxSentinel)
5375 return false;
5376 std::optional<int64_t> LHSImm = getEffectiveImm(MRI, MO: LHS),
5377 RHSImm = getEffectiveImm(MRI, MO: RHS);
5378 if (!LHSImm || !RHSImm)
5379 return false;
5380 return LHSImm <= RHSImm;
5381}
5382
5383namespace {
5384class RISCVPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
5385 const MachineInstr *LHS;
5386 const MachineInstr *RHS;
5387 SmallVector<MachineOperand, 3> Cond;
5388
5389public:
5390 RISCVPipelinerLoopInfo(const MachineInstr *LHS, const MachineInstr *RHS,
5391 const SmallVectorImpl<MachineOperand> &Cond)
5392 : LHS(LHS), RHS(RHS), Cond(Cond.begin(), Cond.end()) {}
5393
5394 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
5395 // Make the instructions for loop control be placed in stage 0.
5396 // The predecessors of LHS/RHS are considered by the caller.
5397 if (LHS && MI == LHS)
5398 return true;
5399 if (RHS && MI == RHS)
5400 return true;
5401 return false;
5402 }
5403
5404 std::optional<bool> createTripCountGreaterCondition(
5405 int TC, MachineBasicBlock &MBB,
5406 SmallVectorImpl<MachineOperand> &CondParam) override {
5407 // A branch instruction will be inserted as "if (Cond) goto epilogue".
5408 // Cond is normalized for such use.
5409 // The predecessors of the branch are assumed to have already been inserted.
5410 CondParam = Cond;
5411 return {};
5412 }
5413
5414 void setPreheader(MachineBasicBlock *NewPreheader) override {}
5415
5416 void adjustTripCount(int TripCountAdjust) override {}
5417};
5418} // namespace
5419
5420std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
5421RISCVInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
5422 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
5423 SmallVector<MachineOperand, 4> Cond;
5424 if (analyzeBranch(MBB&: *LoopBB, TBB, FBB, Cond, /*AllowModify=*/false))
5425 return nullptr;
5426
5427 // Infinite loops are not supported
5428 if (TBB == LoopBB && FBB == LoopBB)
5429 return nullptr;
5430
5431 // Must be conditional branch
5432 if (FBB == nullptr)
5433 return nullptr;
5434
5435 assert((TBB == LoopBB || FBB == LoopBB) &&
5436 "The Loop must be a single-basic-block loop");
5437
5438 // Normalization for createTripCountGreaterCondition()
5439 if (TBB == LoopBB)
5440 reverseBranchCondition(Cond);
5441
5442 const MachineRegisterInfo &MRI = LoopBB->getParent()->getRegInfo();
5443 auto FindRegDef = [&MRI](MachineOperand &Op) -> const MachineInstr * {
5444 if (!Op.isReg())
5445 return nullptr;
5446 Register Reg = Op.getReg();
5447 if (!Reg.isVirtual())
5448 return nullptr;
5449 return MRI.getVRegDef(Reg);
5450 };
5451
5452 const MachineInstr *LHS = FindRegDef(Cond[1]);
5453 const MachineInstr *RHS = FindRegDef(Cond[2]);
5454 if (LHS && LHS->isPHI())
5455 return nullptr;
5456 if (RHS && RHS->isPHI())
5457 return nullptr;
5458
5459 return std::make_unique<RISCVPipelinerLoopInfo>(args&: LHS, args&: RHS, args&: Cond);
5460}
5461
5462// FIXME: We should remove this if we have a default generic scheduling model.
5463bool RISCVInstrInfo::isHighLatencyDef(int Opc) const {
5464 unsigned RVVMCOpcode = RISCV::getRVVMCOpcode(RVVPseudoOpcode: Opc);
5465 Opc = RVVMCOpcode ? RVVMCOpcode : Opc;
5466 switch (Opc) {
5467 default:
5468 return false;
5469 // Integer div/rem.
5470 case RISCV::DIV:
5471 case RISCV::DIVW:
5472 case RISCV::DIVU:
5473 case RISCV::DIVUW:
5474 case RISCV::REM:
5475 case RISCV::REMW:
5476 case RISCV::REMU:
5477 case RISCV::REMUW:
5478 // Floating-point div/sqrt.
5479 case RISCV::FDIV_H:
5480 case RISCV::FDIV_S:
5481 case RISCV::FDIV_D:
5482 case RISCV::FDIV_H_INX:
5483 case RISCV::FDIV_S_INX:
5484 case RISCV::FDIV_D_INX:
5485 case RISCV::FDIV_D_IN32X:
5486 case RISCV::FSQRT_H:
5487 case RISCV::FSQRT_S:
5488 case RISCV::FSQRT_D:
5489 case RISCV::FSQRT_H_INX:
5490 case RISCV::FSQRT_S_INX:
5491 case RISCV::FSQRT_D_INX:
5492 case RISCV::FSQRT_D_IN32X:
5493 // Vector integer div/rem
5494 case RISCV::VDIV_VV:
5495 case RISCV::VDIV_VX:
5496 case RISCV::VDIVU_VV:
5497 case RISCV::VDIVU_VX:
5498 case RISCV::VREM_VV:
5499 case RISCV::VREM_VX:
5500 case RISCV::VREMU_VV:
5501 case RISCV::VREMU_VX:
5502 // Vector floating-point div/sqrt.
5503 case RISCV::VFDIV_VV:
5504 case RISCV::VFDIV_VF:
5505 case RISCV::VFRDIV_VF:
5506 case RISCV::VFSQRT_V:
5507 case RISCV::VFRSQRT7_V:
5508 return true;
5509 }
5510}
5511
5512bool RISCVInstrInfo::isVRegCopy(const MachineInstr *MI, unsigned LMul) const {
5513 if (MI->getOpcode() != TargetOpcode::COPY)
5514 return false;
5515 const MachineRegisterInfo &MRI = MI->getMF()->getRegInfo();
5516 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
5517
5518 Register DstReg = MI->getOperand(i: 0).getReg();
5519 const TargetRegisterClass *RC = DstReg.isVirtual()
5520 ? MRI.getRegClass(Reg: DstReg)
5521 : TRI->getMinimalPhysRegClass(Reg: DstReg);
5522
5523 if (!RISCVRegisterInfo::isRVVRegClass(RC))
5524 return false;
5525
5526 if (!LMul)
5527 return true;
5528
5529 // TODO: Perhaps we could distinguish segment register classes (e.g. VRN3M2)
5530 // in the future.
5531 auto [RCLMul, RCFractional] =
5532 RISCVVType::decodeVLMUL(VLMul: RISCVRI::getLMul(TSFlags: RC->TSFlags));
5533 return (!RCFractional && LMul == RCLMul) || (RCFractional && LMul == 1);
5534}
5535
5536bool RISCVInstrInfo::requiresNTLHint(const MachineInstr &MI) const {
5537 if (MI.memoperands_empty())
5538 return false;
5539
5540 MachineMemOperand *MMO = *(MI.memoperands_begin());
5541 if (!MMO->isNonTemporal())
5542 return false;
5543
5544 return true;
5545}
5546
5547bool RISCVInstrInfo::isSafeToMove(const MachineInstr &From,
5548 const MachineBasicBlock::iterator &To) {
5549 assert(To == From.getParent()->end() || From.getParent() == To->getParent());
5550 SmallVector<Register> PhysUses, PhysDefs;
5551 for (const MachineOperand &MO : From.all_uses())
5552 if (MO.getReg().isPhysical())
5553 PhysUses.push_back(Elt: MO.getReg());
5554 for (const MachineOperand &MO : From.all_defs())
5555 if (MO.getReg().isPhysical())
5556 PhysDefs.push_back(Elt: MO.getReg());
5557 bool SawStore = false;
5558 for (auto II = std::next(x: From.getIterator()); II != To; II++) {
5559 for (Register PhysReg : PhysUses)
5560 if (II->definesRegister(Reg: PhysReg, TRI: nullptr))
5561 return false;
5562 for (Register PhysReg : PhysDefs)
5563 if (II->definesRegister(Reg: PhysReg, TRI: nullptr) ||
5564 II->readsRegister(Reg: PhysReg, TRI: nullptr))
5565 return false;
5566 II->isSafeToMove(SawStore);
5567 if (SawStore)
5568 break;
5569 }
5570 return From.isSafeToMove(SawStore);
5571}
5572