1//===-- RISCVInstrInfo.cpp - RISC-V Instruction Information -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the RISC-V implementation of the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RISCVInstrInfo.h"
14#include "MCTargetDesc/RISCVBaseInfo.h"
15#include "MCTargetDesc/RISCVMatInt.h"
16#include "RISCV.h"
17#include "RISCVMachineFunctionInfo.h"
18#include "RISCVSubtarget.h"
19#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/ADT/Statistic.h"
22#include "llvm/Analysis/MemoryLocation.h"
23#include "llvm/Analysis/ValueTracking.h"
24#include "llvm/CodeGen/LiveIntervals.h"
25#include "llvm/CodeGen/LiveVariables.h"
26#include "llvm/CodeGen/MachineCombinerPattern.h"
27#include "llvm/CodeGen/MachineInstrBuilder.h"
28#include "llvm/CodeGen/MachineRegisterInfo.h"
29#include "llvm/CodeGen/MachineTraceMetrics.h"
30#include "llvm/CodeGen/RegisterScavenging.h"
31#include "llvm/CodeGen/StackMaps.h"
32#include "llvm/IR/DebugInfoMetadata.h"
33#include "llvm/IR/Module.h"
34#include "llvm/MC/MCDwarf.h"
35#include "llvm/MC/MCInstBuilder.h"
36#include "llvm/MC/TargetRegistry.h"
37#include "llvm/Support/ErrorHandling.h"
38
39using namespace llvm;
40
41#define GEN_CHECK_COMPRESS_INSTR
42#include "RISCVGenCompressInstEmitter.inc"
43
44#define GET_INSTRINFO_CTOR_DTOR
45#include "RISCVGenInstrInfo.inc"
46
47#define DEBUG_TYPE "riscv-instr-info"
48STATISTIC(NumVRegSpilled,
49 "Number of registers within vector register groups spilled");
50STATISTIC(NumVRegReloaded,
51 "Number of registers within vector register groups reloaded");
52
53static cl::opt<bool> PreferWholeRegisterMove(
54 "riscv-prefer-whole-register-move", cl::init(Val: false), cl::Hidden,
55 cl::desc("Prefer whole register move for vector registers."));
56
57static cl::opt<MachineTraceStrategy> ForceMachineCombinerStrategy(
58 "riscv-force-machine-combiner-strategy", cl::Hidden,
59 cl::desc("Force machine combiner to use a specific strategy for machine "
60 "trace metrics evaluation."),
61 cl::init(Val: MachineTraceStrategy::TS_NumStrategies),
62 cl::values(clEnumValN(MachineTraceStrategy::TS_Local, "local",
63 "Local strategy."),
64 clEnumValN(MachineTraceStrategy::TS_MinInstrCount, "min-instr",
65 "MinInstrCount strategy.")));
66
67namespace llvm::RISCVVPseudosTable {
68
69using namespace RISCV;
70
71#define GET_RISCVVPseudosTable_IMPL
72#include "RISCVGenSearchableTables.inc"
73
74} // namespace llvm::RISCVVPseudosTable
75
76namespace llvm::RISCV {
77
78#define GET_RISCVMaskedPseudosTable_IMPL
79#include "RISCVGenSearchableTables.inc"
80
81} // end namespace llvm::RISCV
82
83RISCVInstrInfo::RISCVInstrInfo(const RISCVSubtarget &STI)
84 : RISCVGenInstrInfo(STI, RegInfo, RISCV::ADJCALLSTACKDOWN,
85 RISCV::ADJCALLSTACKUP),
86 RegInfo(STI.getHwMode()), STI(STI) {}
87
88#define GET_INSTRINFO_HELPERS
89#include "RISCVGenInstrInfo.inc"
90
91MCInst RISCVInstrInfo::getNop() const {
92 if (STI.hasStdExtZca())
93 return MCInstBuilder(RISCV::C_NOP);
94 return MCInstBuilder(RISCV::ADDI)
95 .addReg(Reg: RISCV::X0)
96 .addReg(Reg: RISCV::X0)
97 .addImm(Val: 0);
98}
99
100Register RISCVInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
101 int &FrameIndex) const {
102 TypeSize Dummy = TypeSize::getZero();
103 return isLoadFromStackSlot(MI, FrameIndex, MemBytes&: Dummy);
104}
105
106static std::optional<unsigned> getLMULForRVVWholeLoadStore(unsigned Opcode) {
107 switch (Opcode) {
108 default:
109 return std::nullopt;
110 case RISCV::VS1R_V:
111 case RISCV::VL1RE8_V:
112 case RISCV::VL1RE16_V:
113 case RISCV::VL1RE32_V:
114 case RISCV::VL1RE64_V:
115 return 1;
116 case RISCV::VS2R_V:
117 case RISCV::VL2RE8_V:
118 case RISCV::VL2RE16_V:
119 case RISCV::VL2RE32_V:
120 case RISCV::VL2RE64_V:
121 return 2;
122 case RISCV::VS4R_V:
123 case RISCV::VL4RE8_V:
124 case RISCV::VL4RE16_V:
125 case RISCV::VL4RE32_V:
126 case RISCV::VL4RE64_V:
127 return 4;
128 case RISCV::VS8R_V:
129 case RISCV::VL8RE8_V:
130 case RISCV::VL8RE16_V:
131 case RISCV::VL8RE32_V:
132 case RISCV::VL8RE64_V:
133 return 8;
134 }
135}
136
137Register RISCVInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
138 int &FrameIndex,
139 TypeSize &MemBytes) const {
140 switch (MI.getOpcode()) {
141 default:
142 return 0;
143 case RISCV::LB:
144 case RISCV::LBU:
145 MemBytes = TypeSize::getFixed(ExactSize: 1);
146 break;
147 case RISCV::LH:
148 case RISCV::LH_INX:
149 case RISCV::LHU:
150 case RISCV::FLH:
151 MemBytes = TypeSize::getFixed(ExactSize: 2);
152 break;
153 case RISCV::LW:
154 case RISCV::LW_INX:
155 case RISCV::FLW:
156 case RISCV::LWU:
157 MemBytes = TypeSize::getFixed(ExactSize: 4);
158 break;
159 case RISCV::LD:
160 case RISCV::LD_RV32:
161 case RISCV::FLD:
162 MemBytes = TypeSize::getFixed(ExactSize: 8);
163 break;
164 case RISCV::VL1RE8_V:
165 case RISCV::VL2RE8_V:
166 case RISCV::VL4RE8_V:
167 case RISCV::VL8RE8_V:
168 if (!MI.getOperand(i: 1).isFI())
169 return Register();
170 FrameIndex = MI.getOperand(i: 1).getIndex();
171 unsigned LMUL = *getLMULForRVVWholeLoadStore(Opcode: MI.getOpcode());
172 MemBytes = TypeSize::getScalable(MinimumSize: RISCV::RVVBytesPerBlock * LMUL);
173 return MI.getOperand(i: 0).getReg();
174 }
175
176 if (MI.getOperand(i: 1).isFI() && MI.getOperand(i: 2).isImm() &&
177 MI.getOperand(i: 2).getImm() == 0) {
178 FrameIndex = MI.getOperand(i: 1).getIndex();
179 return MI.getOperand(i: 0).getReg();
180 }
181
182 return 0;
183}
184
185Register RISCVInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
186 int &FrameIndex) const {
187 TypeSize Dummy = TypeSize::getZero();
188 return isStoreToStackSlot(MI, FrameIndex, MemBytes&: Dummy);
189}
190
191Register RISCVInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
192 int &FrameIndex,
193 TypeSize &MemBytes) const {
194 switch (MI.getOpcode()) {
195 default:
196 return 0;
197 case RISCV::SB:
198 MemBytes = TypeSize::getFixed(ExactSize: 1);
199 break;
200 case RISCV::SH:
201 case RISCV::SH_INX:
202 case RISCV::FSH:
203 MemBytes = TypeSize::getFixed(ExactSize: 2);
204 break;
205 case RISCV::SW:
206 case RISCV::SW_INX:
207 case RISCV::FSW:
208 MemBytes = TypeSize::getFixed(ExactSize: 4);
209 break;
210 case RISCV::SD:
211 case RISCV::SD_RV32:
212 case RISCV::FSD:
213 MemBytes = TypeSize::getFixed(ExactSize: 8);
214 break;
215 case RISCV::VS1R_V:
216 case RISCV::VS2R_V:
217 case RISCV::VS4R_V:
218 case RISCV::VS8R_V:
219 if (!MI.getOperand(i: 1).isFI())
220 return Register();
221 FrameIndex = MI.getOperand(i: 1).getIndex();
222 unsigned LMUL = *getLMULForRVVWholeLoadStore(Opcode: MI.getOpcode());
223 MemBytes = TypeSize::getScalable(MinimumSize: RISCV::RVVBytesPerBlock * LMUL);
224 return MI.getOperand(i: 0).getReg();
225 }
226
227 if (MI.getOperand(i: 1).isFI() && MI.getOperand(i: 2).isImm() &&
228 MI.getOperand(i: 2).getImm() == 0) {
229 FrameIndex = MI.getOperand(i: 1).getIndex();
230 return MI.getOperand(i: 0).getReg();
231 }
232
233 return 0;
234}
235
236bool RISCVInstrInfo::isReMaterializableImpl(
237 const MachineInstr &MI) const {
238 switch (RISCV::getRVVMCOpcode(RVVPseudoOpcode: MI.getOpcode())) {
239 case RISCV::VMV_V_X:
240 case RISCV::VFMV_V_F:
241 case RISCV::VMV_V_I:
242 case RISCV::VMV_S_X:
243 case RISCV::VFMV_S_F:
244 case RISCV::VID_V:
245 return MI.getOperand(i: 1).isUndef();
246 default:
247 return TargetInstrInfo::isReMaterializableImpl(MI);
248 }
249}
250
251static bool forwardCopyWillClobberTuple(unsigned DstReg, unsigned SrcReg,
252 unsigned NumRegs) {
253 return DstReg > SrcReg && (DstReg - SrcReg) < NumRegs;
254}
255
256static bool isConvertibleToVMV_V_V(const RISCVSubtarget &STI,
257 const MachineBasicBlock &MBB,
258 MachineBasicBlock::const_iterator MBBI,
259 MachineBasicBlock::const_iterator &DefMBBI,
260 RISCVVType::VLMUL LMul) {
261 if (PreferWholeRegisterMove)
262 return false;
263
264 assert(MBBI->getOpcode() == TargetOpcode::COPY &&
265 "Unexpected COPY instruction.");
266 Register SrcReg = MBBI->getOperand(i: 1).getReg();
267 const TargetRegisterInfo *TRI = STI.getRegisterInfo();
268
269 bool FoundDef = false;
270 bool FirstVSetVLI = false;
271 unsigned FirstSEW = 0;
272 while (MBBI != MBB.begin()) {
273 --MBBI;
274 if (MBBI->isMetaInstruction())
275 continue;
276
277 if (RISCVInstrInfo::isVectorConfigInstr(MI: *MBBI)) {
278 // There is a vsetvli between COPY and source define instruction.
279 // vy = def_vop ... (producing instruction)
280 // ...
281 // vsetvli
282 // ...
283 // vx = COPY vy
284 if (!FoundDef) {
285 if (!FirstVSetVLI) {
286 FirstVSetVLI = true;
287 unsigned FirstVType = MBBI->getOperand(i: 2).getImm();
288 RISCVVType::VLMUL FirstLMul = RISCVVType::getVLMUL(VType: FirstVType);
289 FirstSEW = RISCVVType::getSEW(VType: FirstVType);
290 // The first encountered vsetvli must have the same lmul as the
291 // register class of COPY.
292 if (FirstLMul != LMul)
293 return false;
294 }
295 // Only permit `vsetvli x0, x0, vtype` between COPY and the source
296 // define instruction.
297 if (!RISCVInstrInfo::isVLPreservingConfig(MI: *MBBI))
298 return false;
299 continue;
300 }
301
302 // MBBI is the first vsetvli before the producing instruction.
303 unsigned VType = MBBI->getOperand(i: 2).getImm();
304 // If there is a vsetvli between COPY and the producing instruction.
305 if (FirstVSetVLI) {
306 // If SEW is different, return false.
307 if (RISCVVType::getSEW(VType) != FirstSEW)
308 return false;
309 }
310
311 // If the vsetvli is tail undisturbed, keep the whole register move.
312 if (!RISCVVType::isTailAgnostic(VType))
313 return false;
314
315 // The checking is conservative. We only have register classes for
316 // LMUL = 1/2/4/8. We should be able to convert vmv1r.v to vmv.v.v
317 // for fractional LMUL operations. However, we could not use the vsetvli
318 // lmul for widening operations. The result of widening operation is
319 // 2 x LMUL.
320 return LMul == RISCVVType::getVLMUL(VType);
321 } else if (MBBI->isInlineAsm() || MBBI->isCall()) {
322 return false;
323 } else if (MBBI->getNumDefs()) {
324 // Check all the instructions which will change VL.
325 // For example, vleff has implicit def VL.
326 if (MBBI->modifiesRegister(Reg: RISCV::VL, /*TRI=*/nullptr))
327 return false;
328
329 // Only converting whole register copies to vmv.v.v when the defining
330 // value appears in the explicit operands.
331 for (const MachineOperand &MO : MBBI->explicit_operands()) {
332 if (!MO.isReg() || !MO.isDef())
333 continue;
334 if (!FoundDef && TRI->regsOverlap(RegA: MO.getReg(), RegB: SrcReg)) {
335 // We only permit the source of COPY has the same LMUL as the defined
336 // operand.
337 // There are cases we need to keep the whole register copy if the LMUL
338 // is different.
339 // For example,
340 // $x0 = PseudoVSETIVLI 4, 73 // vsetivli zero, 4, e16,m2,ta,m
341 // $v28m4 = PseudoVWADD_VV_M2 $v26m2, $v8m2
342 // # The COPY may be created by vlmul_trunc intrinsic.
343 // $v26m2 = COPY renamable $v28m2, implicit killed $v28m4
344 //
345 // After widening, the valid value will be 4 x e32 elements. If we
346 // convert the COPY to vmv.v.v, it will only copy 4 x e16 elements.
347 // FIXME: The COPY of subregister of Zvlsseg register will not be able
348 // to convert to vmv.v.[v|i] under the constraint.
349 if (MO.getReg() != SrcReg)
350 return false;
351
352 // In widening reduction instructions with LMUL_1 input vector case,
353 // only checking the LMUL is insufficient due to reduction result is
354 // always LMUL_1.
355 // For example,
356 // $x11 = PseudoVSETIVLI 1, 64 // vsetivli a1, 1, e8, m1, ta, mu
357 // $v8m1 = PseudoVWREDSUM_VS_M1 $v26, $v27
358 // $v26 = COPY killed renamable $v8
359 // After widening, The valid value will be 1 x e16 elements. If we
360 // convert the COPY to vmv.v.v, it will only copy 1 x e8 elements.
361 uint64_t TSFlags = MBBI->getDesc().TSFlags;
362 if (RISCVII::isRVVWideningReduction(TSFlags))
363 return false;
364
365 // If the producing instruction does not depend on vsetvli, do not
366 // convert COPY to vmv.v.v. For example, VL1R_V or PseudoVRELOAD.
367 if (!RISCVII::hasSEWOp(TSFlags) || !RISCVII::hasVLOp(TSFlags))
368 return false;
369
370 // Found the definition.
371 FoundDef = true;
372 DefMBBI = MBBI;
373 break;
374 }
375 }
376 }
377 }
378
379 return false;
380}
381
382void RISCVInstrInfo::copyPhysRegVector(
383 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
384 const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc,
385 const TargetRegisterClass *RegClass) const {
386 const RISCVRegisterInfo *TRI = STI.getRegisterInfo();
387 RISCVVType::VLMUL LMul = RISCVRI::getLMul(TSFlags: RegClass->TSFlags);
388 unsigned NF = RISCVRI::getNF(TSFlags: RegClass->TSFlags);
389
390 uint16_t SrcEncoding = TRI->getEncodingValue(Reg: SrcReg);
391 uint16_t DstEncoding = TRI->getEncodingValue(Reg: DstReg);
392 auto [LMulVal, Fractional] = RISCVVType::decodeVLMUL(VLMul: LMul);
393 assert(!Fractional && "It is impossible be fractional lmul here.");
394 unsigned NumRegs = NF * LMulVal;
395 bool ReversedCopy =
396 forwardCopyWillClobberTuple(DstReg: DstEncoding, SrcReg: SrcEncoding, NumRegs);
397 if (ReversedCopy) {
398 // If the src and dest overlap when copying a tuple, we need to copy the
399 // registers in reverse.
400 SrcEncoding += NumRegs - 1;
401 DstEncoding += NumRegs - 1;
402 }
403
404 unsigned I = 0;
405 auto GetCopyInfo = [&](uint16_t SrcEncoding, uint16_t DstEncoding)
406 -> std::tuple<RISCVVType::VLMUL, const TargetRegisterClass &, unsigned,
407 unsigned, unsigned> {
408 if (ReversedCopy) {
409 // For reversed copying, if there are enough aligned registers(8/4/2), we
410 // can do a larger copy(LMUL8/4/2).
411 // Besides, we have already known that DstEncoding is larger than
412 // SrcEncoding in forwardCopyWillClobberTuple, so the difference between
413 // DstEncoding and SrcEncoding should be >= LMUL value we try to use to
414 // avoid clobbering.
415 uint16_t Diff = DstEncoding - SrcEncoding;
416 if (I + 8 <= NumRegs && Diff >= 8 && SrcEncoding % 8 == 7 &&
417 DstEncoding % 8 == 7)
418 return {RISCVVType::LMUL_8, RISCV::VRM8RegClass, RISCV::VMV8R_V,
419 RISCV::PseudoVMV_V_V_M8, RISCV::PseudoVMV_V_I_M8};
420 if (I + 4 <= NumRegs && Diff >= 4 && SrcEncoding % 4 == 3 &&
421 DstEncoding % 4 == 3)
422 return {RISCVVType::LMUL_4, RISCV::VRM4RegClass, RISCV::VMV4R_V,
423 RISCV::PseudoVMV_V_V_M4, RISCV::PseudoVMV_V_I_M4};
424 if (I + 2 <= NumRegs && Diff >= 2 && SrcEncoding % 2 == 1 &&
425 DstEncoding % 2 == 1)
426 return {RISCVVType::LMUL_2, RISCV::VRM2RegClass, RISCV::VMV2R_V,
427 RISCV::PseudoVMV_V_V_M2, RISCV::PseudoVMV_V_I_M2};
428 // Or we should do LMUL1 copying.
429 return {RISCVVType::LMUL_1, RISCV::VRRegClass, RISCV::VMV1R_V,
430 RISCV::PseudoVMV_V_V_M1, RISCV::PseudoVMV_V_I_M1};
431 }
432
433 // For forward copying, if source register encoding and destination register
434 // encoding are aligned to 8/4/2, we can do a LMUL8/4/2 copying.
435 if (I + 8 <= NumRegs && SrcEncoding % 8 == 0 && DstEncoding % 8 == 0)
436 return {RISCVVType::LMUL_8, RISCV::VRM8RegClass, RISCV::VMV8R_V,
437 RISCV::PseudoVMV_V_V_M8, RISCV::PseudoVMV_V_I_M8};
438 if (I + 4 <= NumRegs && SrcEncoding % 4 == 0 && DstEncoding % 4 == 0)
439 return {RISCVVType::LMUL_4, RISCV::VRM4RegClass, RISCV::VMV4R_V,
440 RISCV::PseudoVMV_V_V_M4, RISCV::PseudoVMV_V_I_M4};
441 if (I + 2 <= NumRegs && SrcEncoding % 2 == 0 && DstEncoding % 2 == 0)
442 return {RISCVVType::LMUL_2, RISCV::VRM2RegClass, RISCV::VMV2R_V,
443 RISCV::PseudoVMV_V_V_M2, RISCV::PseudoVMV_V_I_M2};
444 // Or we should do LMUL1 copying.
445 return {RISCVVType::LMUL_1, RISCV::VRRegClass, RISCV::VMV1R_V,
446 RISCV::PseudoVMV_V_V_M1, RISCV::PseudoVMV_V_I_M1};
447 };
448
449 while (I != NumRegs) {
450 // For non-segment copying, we only do this once as the registers are always
451 // aligned.
452 // For segment copying, we may do this several times. If the registers are
453 // aligned to larger LMUL, we can eliminate some copyings.
454 auto [LMulCopied, RegClass, Opc, VVOpc, VIOpc] =
455 GetCopyInfo(SrcEncoding, DstEncoding);
456 auto [NumCopied, _] = RISCVVType::decodeVLMUL(VLMul: LMulCopied);
457
458 MachineBasicBlock::const_iterator DefMBBI;
459 if (LMul == LMulCopied &&
460 isConvertibleToVMV_V_V(STI, MBB, MBBI, DefMBBI, LMul)) {
461 Opc = VVOpc;
462 if (DefMBBI->getOpcode() == VIOpc)
463 Opc = VIOpc;
464 }
465
466 // Emit actual copying.
467 // For reversed copying, the encoding should be decreased.
468 MCRegister ActualSrcReg = TRI->findVRegWithEncoding(
469 RegClass, Encoding: ReversedCopy ? (SrcEncoding - NumCopied + 1) : SrcEncoding);
470 MCRegister ActualDstReg = TRI->findVRegWithEncoding(
471 RegClass, Encoding: ReversedCopy ? (DstEncoding - NumCopied + 1) : DstEncoding);
472
473 auto MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: Opc), DestReg: ActualDstReg);
474 bool UseVMV_V_I = RISCV::getRVVMCOpcode(RVVPseudoOpcode: Opc) == RISCV::VMV_V_I;
475 bool UseVMV = UseVMV_V_I || RISCV::getRVVMCOpcode(RVVPseudoOpcode: Opc) == RISCV::VMV_V_V;
476 if (UseVMV)
477 MIB.addReg(RegNo: ActualDstReg, Flags: RegState::Undef);
478 if (UseVMV_V_I)
479 MIB = MIB.add(MO: DefMBBI->getOperand(i: 2));
480 else
481 MIB = MIB.addReg(RegNo: ActualSrcReg, Flags: getKillRegState(B: KillSrc));
482 if (UseVMV) {
483 const MCInstrDesc &Desc = DefMBBI->getDesc();
484 MIB.add(MO: DefMBBI->getOperand(i: RISCVII::getVLOpNum(Desc))); // AVL
485 unsigned Log2SEW =
486 DefMBBI->getOperand(i: RISCVII::getSEWOpNum(Desc)).getImm();
487 MIB.addImm(Val: Log2SEW ? Log2SEW : 3); // SEW
488 MIB.addImm(Val: 0); // tu, mu
489 MIB.addReg(RegNo: RISCV::VL, Flags: RegState::Implicit);
490 MIB.addReg(RegNo: RISCV::VTYPE, Flags: RegState::Implicit);
491 }
492 // Add an implicit read of the original source to silence the verifier
493 // in the cases where some of the smaller VRs we're copying from might be
494 // undef, caused by the fact that the original, larger source VR might not
495 // be fully initialized at the time this COPY happens.
496 MIB.addReg(RegNo: SrcReg, Flags: RegState::Implicit);
497
498 // If we are copying reversely, we should decrease the encoding.
499 SrcEncoding += (ReversedCopy ? -NumCopied : NumCopied);
500 DstEncoding += (ReversedCopy ? -NumCopied : NumCopied);
501 I += NumCopied;
502 }
503}
504
505void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
506 MachineBasicBlock::iterator MBBI,
507 const DebugLoc &DL, Register DstReg,
508 Register SrcReg, bool KillSrc,
509 bool RenamableDest, bool RenamableSrc) const {
510 const TargetRegisterInfo *TRI = STI.getRegisterInfo();
511 RegState KillFlag = getKillRegState(B: KillSrc);
512
513 if (RISCV::GPRRegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
514 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::ADDI), DestReg: DstReg)
515 .addReg(RegNo: SrcReg, Flags: KillFlag | getRenamableRegState(B: RenamableSrc))
516 .addImm(Val: 0);
517 return;
518 }
519
520 if (RISCV::GPRF16RegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
521 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::PseudoMV_FPR16INX), DestReg: DstReg)
522 .addReg(RegNo: SrcReg, Flags: KillFlag | getRenamableRegState(B: RenamableSrc));
523 return;
524 }
525
526 if (RISCV::GPRF32RegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
527 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::PseudoMV_FPR32INX), DestReg: DstReg)
528 .addReg(RegNo: SrcReg, Flags: KillFlag | getRenamableRegState(B: RenamableSrc));
529 return;
530 }
531
532 if (RISCV::GPRPairRegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
533 if (STI.isRV32()) {
534 if (STI.hasStdExtZdinx()) {
535 // On RV32_Zdinx, FMV.D will move a pair of registers to another pair of
536 // registers, in one instruction.
537 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FSGNJ_D_IN32X), DestReg: DstReg)
538 .addReg(RegNo: SrcReg, Flags: getRenamableRegState(B: RenamableSrc))
539 .addReg(RegNo: SrcReg, Flags: KillFlag | getRenamableRegState(B: RenamableSrc));
540 return;
541 }
542
543 if (STI.hasStdExtP()) {
544 // On RV32P, `padd.dw` is a GPR Pair Add
545 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::PADD_DW), DestReg: DstReg)
546 .addReg(RegNo: SrcReg, Flags: KillFlag | getRenamableRegState(B: RenamableSrc))
547 .addReg(RegNo: RISCV::X0_Pair);
548 return;
549 }
550 }
551
552 MCRegister EvenReg = TRI->getSubReg(Reg: SrcReg, Idx: RISCV::sub_gpr_even);
553 MCRegister OddReg = TRI->getSubReg(Reg: SrcReg, Idx: RISCV::sub_gpr_odd);
554 // We need to correct the odd register of X0_Pair.
555 if (OddReg == RISCV::DUMMY_REG_PAIR_WITH_X0)
556 OddReg = RISCV::X0;
557 assert(DstReg != RISCV::X0_Pair && "Cannot write to X0_Pair");
558
559 // Emit an ADDI for both parts of GPRPair.
560 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::ADDI),
561 DestReg: TRI->getSubReg(Reg: DstReg, Idx: RISCV::sub_gpr_even))
562 .addReg(RegNo: EvenReg, Flags: KillFlag)
563 .addImm(Val: 0);
564 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::ADDI),
565 DestReg: TRI->getSubReg(Reg: DstReg, Idx: RISCV::sub_gpr_odd))
566 .addReg(RegNo: OddReg, Flags: KillFlag)
567 .addImm(Val: 0);
568 return;
569 }
570
571 // Handle copy from csr
572 if (RISCV::VCSRRegClass.contains(Reg: SrcReg) &&
573 RISCV::GPRRegClass.contains(Reg: DstReg)) {
574 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::CSRRS), DestReg: DstReg)
575 .addImm(Val: RISCVSysReg::lookupSysRegByName(Name: TRI->getName(RegNo: SrcReg))->Encoding)
576 .addReg(RegNo: RISCV::X0);
577 return;
578 }
579
580 if (RISCV::FPR16RegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
581 unsigned Opc;
582 if (STI.hasStdExtZfh()) {
583 Opc = RISCV::FSGNJ_H;
584 } else {
585 assert(STI.hasStdExtF() &&
586 (STI.hasStdExtZfhmin() || STI.hasStdExtZfbfmin()) &&
587 "Unexpected extensions");
588 // Zfhmin/Zfbfmin doesn't have FSGNJ_H, replace FSGNJ_H with FSGNJ_S.
589 DstReg = TRI->getMatchingSuperReg(Reg: DstReg, SubIdx: RISCV::sub_16,
590 RC: &RISCV::FPR32RegClass);
591 SrcReg = TRI->getMatchingSuperReg(Reg: SrcReg, SubIdx: RISCV::sub_16,
592 RC: &RISCV::FPR32RegClass);
593 Opc = RISCV::FSGNJ_S;
594 }
595 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: Opc), DestReg: DstReg)
596 .addReg(RegNo: SrcReg, Flags: KillFlag)
597 .addReg(RegNo: SrcReg, Flags: KillFlag);
598 return;
599 }
600
601 if (RISCV::FPR32RegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
602 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FSGNJ_S), DestReg: DstReg)
603 .addReg(RegNo: SrcReg, Flags: KillFlag)
604 .addReg(RegNo: SrcReg, Flags: KillFlag);
605 return;
606 }
607
608 if (RISCV::FPR64RegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
609 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FSGNJ_D), DestReg: DstReg)
610 .addReg(RegNo: SrcReg, Flags: KillFlag)
611 .addReg(RegNo: SrcReg, Flags: KillFlag);
612 return;
613 }
614
615 if (RISCV::FPR32RegClass.contains(Reg: DstReg) &&
616 RISCV::GPRRegClass.contains(Reg: SrcReg)) {
617 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FMV_W_X), DestReg: DstReg)
618 .addReg(RegNo: SrcReg, Flags: KillFlag);
619 return;
620 }
621
622 if (RISCV::GPRRegClass.contains(Reg: DstReg) &&
623 RISCV::FPR32RegClass.contains(Reg: SrcReg)) {
624 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FMV_X_W), DestReg: DstReg)
625 .addReg(RegNo: SrcReg, Flags: KillFlag);
626 return;
627 }
628
629 if (RISCV::FPR64RegClass.contains(Reg: DstReg) &&
630 RISCV::GPRRegClass.contains(Reg: SrcReg)) {
631 assert(STI.getXLen() == 64 && "Unexpected GPR size");
632 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FMV_D_X), DestReg: DstReg)
633 .addReg(RegNo: SrcReg, Flags: KillFlag);
634 return;
635 }
636
637 if (RISCV::GPRRegClass.contains(Reg: DstReg) &&
638 RISCV::FPR64RegClass.contains(Reg: SrcReg)) {
639 assert(STI.getXLen() == 64 && "Unexpected GPR size");
640 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FMV_X_D), DestReg: DstReg)
641 .addReg(RegNo: SrcReg, Flags: KillFlag);
642 return;
643 }
644
645 // VR->VR copies.
646 const TargetRegisterClass *RegClass =
647 TRI->getCommonMinimalPhysRegClass(Reg1: SrcReg, Reg2: DstReg);
648 if (RISCVRegisterInfo::isRVVRegClass(RC: RegClass)) {
649 copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RegClass);
650 return;
651 }
652
653 llvm_unreachable("Impossible reg-to-reg copy");
654}
655
656void RISCVInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
657 MachineBasicBlock::iterator I,
658 Register SrcReg, bool IsKill, int FI,
659 const TargetRegisterClass *RC,
660 Register VReg,
661 MachineInstr::MIFlag Flags) const {
662 MachineFunction *MF = MBB.getParent();
663 MachineFrameInfo &MFI = MF->getFrameInfo();
664 Align Alignment = MFI.getObjectAlign(ObjectIdx: FI);
665
666 unsigned Opcode;
667 if (RISCV::GPRRegClass.hasSubClassEq(RC)) {
668 Opcode = RegInfo.getRegSizeInBits(RC: RISCV::GPRRegClass) == 32 ? RISCV::SW
669 : RISCV::SD;
670 } else if (RISCV::GPRF16RegClass.hasSubClassEq(RC)) {
671 Opcode = RISCV::SH_INX;
672 } else if (RISCV::GPRF32RegClass.hasSubClassEq(RC)) {
673 Opcode = RISCV::SW_INX;
674 } else if (RISCV::GPRPairRegClass.hasSubClassEq(RC)) {
675 if (!STI.is64Bit() && STI.hasStdExtZilsd() &&
676 Alignment >= STI.getZilsdAlign()) {
677 Opcode = RISCV::SD_RV32;
678 } else {
679 Opcode = RISCV::PseudoRV32ZdinxSD;
680 }
681 } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) {
682 Opcode = RISCV::FSH;
683 } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) {
684 Opcode = RISCV::FSW;
685 } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) {
686 Opcode = RISCV::FSD;
687 } else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
688 Opcode = RISCV::VS1R_V;
689 } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
690 Opcode = RISCV::VS2R_V;
691 } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
692 Opcode = RISCV::VS4R_V;
693 } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
694 Opcode = RISCV::VS8R_V;
695 } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC))
696 Opcode = RISCV::PseudoVSPILL2_M1;
697 else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC))
698 Opcode = RISCV::PseudoVSPILL2_M2;
699 else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC))
700 Opcode = RISCV::PseudoVSPILL2_M4;
701 else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC))
702 Opcode = RISCV::PseudoVSPILL3_M1;
703 else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC))
704 Opcode = RISCV::PseudoVSPILL3_M2;
705 else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC))
706 Opcode = RISCV::PseudoVSPILL4_M1;
707 else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC))
708 Opcode = RISCV::PseudoVSPILL4_M2;
709 else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC))
710 Opcode = RISCV::PseudoVSPILL5_M1;
711 else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC))
712 Opcode = RISCV::PseudoVSPILL6_M1;
713 else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC))
714 Opcode = RISCV::PseudoVSPILL7_M1;
715 else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC))
716 Opcode = RISCV::PseudoVSPILL8_M1;
717 else
718 llvm_unreachable("Can't store this register to stack slot");
719
720 if (RISCVRegisterInfo::isRVVRegClass(RC)) {
721 MachineMemOperand *MMO = MF->getMachineMemOperand(
722 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), F: MachineMemOperand::MOStore,
723 Size: TypeSize::getScalable(MinimumSize: MFI.getObjectSize(ObjectIdx: FI)), BaseAlignment: Alignment);
724
725 MFI.setStackID(ObjectIdx: FI, ID: TargetStackID::ScalableVector);
726 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode))
727 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: IsKill))
728 .addFrameIndex(Idx: FI)
729 .addMemOperand(MMO)
730 .setMIFlag(Flags);
731 NumVRegSpilled += RegInfo.getRegSizeInBits(RC: *RC) / RISCV::RVVBitsPerBlock;
732 } else {
733 MachineMemOperand *MMO = MF->getMachineMemOperand(
734 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), F: MachineMemOperand::MOStore,
735 Size: MFI.getObjectSize(ObjectIdx: FI), BaseAlignment: Alignment);
736
737 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode))
738 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: IsKill))
739 .addFrameIndex(Idx: FI)
740 .addImm(Val: 0)
741 .addMemOperand(MMO)
742 .setMIFlag(Flags);
743 }
744}
745
746void RISCVInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
747 MachineBasicBlock::iterator I,
748 Register DstReg, int FI,
749 const TargetRegisterClass *RC,
750 Register VReg, unsigned SubReg,
751 MachineInstr::MIFlag Flags) const {
752 MachineFunction *MF = MBB.getParent();
753 MachineFrameInfo &MFI = MF->getFrameInfo();
754 Align Alignment = MFI.getObjectAlign(ObjectIdx: FI);
755 DebugLoc DL =
756 Flags & MachineInstr::FrameDestroy ? MBB.findDebugLoc(MBBI: I) : DebugLoc();
757
758 unsigned Opcode;
759 if (RISCV::GPRRegClass.hasSubClassEq(RC)) {
760 Opcode = RegInfo.getRegSizeInBits(RC: RISCV::GPRRegClass) == 32 ? RISCV::LW
761 : RISCV::LD;
762 } else if (RISCV::GPRF16RegClass.hasSubClassEq(RC)) {
763 Opcode = RISCV::LH_INX;
764 } else if (RISCV::GPRF32RegClass.hasSubClassEq(RC)) {
765 Opcode = RISCV::LW_INX;
766 } else if (RISCV::GPRPairRegClass.hasSubClassEq(RC)) {
767 if (!STI.is64Bit() && STI.hasStdExtZilsd() &&
768 Alignment >= STI.getZilsdAlign()) {
769 Opcode = RISCV::LD_RV32;
770 } else {
771 Opcode = RISCV::PseudoRV32ZdinxLD;
772 }
773 } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) {
774 Opcode = RISCV::FLH;
775 } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) {
776 Opcode = RISCV::FLW;
777 } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) {
778 Opcode = RISCV::FLD;
779 } else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
780 Opcode = RISCV::VL1RE8_V;
781 } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
782 Opcode = RISCV::VL2RE8_V;
783 } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
784 Opcode = RISCV::VL4RE8_V;
785 } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
786 Opcode = RISCV::VL8RE8_V;
787 } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC))
788 Opcode = RISCV::PseudoVRELOAD2_M1;
789 else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC))
790 Opcode = RISCV::PseudoVRELOAD2_M2;
791 else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC))
792 Opcode = RISCV::PseudoVRELOAD2_M4;
793 else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC))
794 Opcode = RISCV::PseudoVRELOAD3_M1;
795 else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC))
796 Opcode = RISCV::PseudoVRELOAD3_M2;
797 else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC))
798 Opcode = RISCV::PseudoVRELOAD4_M1;
799 else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC))
800 Opcode = RISCV::PseudoVRELOAD4_M2;
801 else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC))
802 Opcode = RISCV::PseudoVRELOAD5_M1;
803 else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC))
804 Opcode = RISCV::PseudoVRELOAD6_M1;
805 else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC))
806 Opcode = RISCV::PseudoVRELOAD7_M1;
807 else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC))
808 Opcode = RISCV::PseudoVRELOAD8_M1;
809 else
810 llvm_unreachable("Can't load this register from stack slot");
811
812 if (RISCVRegisterInfo::isRVVRegClass(RC)) {
813 MachineMemOperand *MMO = MF->getMachineMemOperand(
814 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), F: MachineMemOperand::MOLoad,
815 Size: TypeSize::getScalable(MinimumSize: MFI.getObjectSize(ObjectIdx: FI)), BaseAlignment: Alignment);
816
817 MFI.setStackID(ObjectIdx: FI, ID: TargetStackID::ScalableVector);
818 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode), DestReg: DstReg)
819 .addFrameIndex(Idx: FI)
820 .addMemOperand(MMO)
821 .setMIFlag(Flags);
822 NumVRegReloaded += RegInfo.getRegSizeInBits(RC: *RC) / RISCV::RVVBitsPerBlock;
823 } else {
824 MachineMemOperand *MMO = MF->getMachineMemOperand(
825 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), F: MachineMemOperand::MOLoad,
826 Size: MFI.getObjectSize(ObjectIdx: FI), BaseAlignment: Alignment);
827
828 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode), DestReg: DstReg)
829 .addFrameIndex(Idx: FI)
830 .addImm(Val: 0)
831 .addMemOperand(MMO)
832 .setMIFlag(Flags);
833 }
834}
835std::optional<unsigned> getFoldedOpcode(MachineFunction &MF, MachineInstr &MI,
836 ArrayRef<unsigned> Ops,
837 const RISCVSubtarget &ST) {
838
839 // The below optimizations narrow the load so they are only valid for little
840 // endian.
841 // TODO: Support big endian by adding an offset into the frame object?
842 if (MF.getDataLayout().isBigEndian())
843 return std::nullopt;
844
845 // Fold load from stack followed by sext.b/sext.h/sext.w/zext.b/zext.h/zext.w.
846 if (Ops.size() != 1 || Ops[0] != 1)
847 return std::nullopt;
848
849 switch (MI.getOpcode()) {
850 default:
851 if (RISCVInstrInfo::isSEXT_W(MI))
852 return RISCV::LW;
853 if (RISCVInstrInfo::isZEXT_W(MI))
854 return RISCV::LWU;
855 if (RISCVInstrInfo::isZEXT_B(MI))
856 return RISCV::LBU;
857 break;
858 case RISCV::SEXT_H:
859 return RISCV::LH;
860 case RISCV::SEXT_B:
861 return RISCV::LB;
862 case RISCV::ZEXT_H_RV32:
863 case RISCV::ZEXT_H_RV64:
864 return RISCV::LHU;
865 }
866
867 switch (RISCV::getRVVMCOpcode(RVVPseudoOpcode: MI.getOpcode())) {
868 default:
869 return std::nullopt;
870 case RISCV::VMV_X_S: {
871 unsigned Log2SEW =
872 MI.getOperand(i: RISCVII::getSEWOpNum(Desc: MI.getDesc())).getImm();
873 if (ST.getXLen() < (1U << Log2SEW))
874 return std::nullopt;
875 switch (Log2SEW) {
876 case 3:
877 return RISCV::LB;
878 case 4:
879 return RISCV::LH;
880 case 5:
881 return RISCV::LW;
882 case 6:
883 return RISCV::LD;
884 default:
885 llvm_unreachable("Unexpected SEW");
886 }
887 }
888 case RISCV::VFMV_F_S: {
889 unsigned Log2SEW =
890 MI.getOperand(i: RISCVII::getSEWOpNum(Desc: MI.getDesc())).getImm();
891 switch (Log2SEW) {
892 case 4:
893 return RISCV::FLH;
894 case 5:
895 return RISCV::FLW;
896 case 6:
897 return RISCV::FLD;
898 default:
899 llvm_unreachable("Unexpected SEW");
900 }
901 }
902 }
903}
904
905// This is the version used during InlineSpiller::spillAroundUses
906MachineInstr *RISCVInstrInfo::foldMemoryOperandImpl(
907 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
908 MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS,
909 VirtRegMap *VRM) const {
910
911 std::optional<unsigned> LoadOpc = getFoldedOpcode(MF, MI, Ops, ST: STI);
912 if (!LoadOpc)
913 return nullptr;
914 Register DstReg = MI.getOperand(i: 0).getReg();
915 return BuildMI(BB&: *MI.getParent(), I: InsertPt, MIMD: MI.getDebugLoc(), MCID: get(Opcode: *LoadOpc),
916 DestReg: DstReg)
917 .addFrameIndex(Idx: FrameIndex)
918 .addImm(Val: 0);
919}
920
921static unsigned getLoadPredicatedOpcode(unsigned Opcode) {
922 switch (Opcode) {
923 case RISCV::LB:
924 return RISCV::PseudoCCLB;
925 case RISCV::LBU:
926 return RISCV::PseudoCCLBU;
927 case RISCV::LH:
928 return RISCV::PseudoCCLH;
929 case RISCV::LHU:
930 return RISCV::PseudoCCLHU;
931 case RISCV::LW:
932 return RISCV::PseudoCCLW;
933 case RISCV::LWU:
934 return RISCV::PseudoCCLWU;
935 case RISCV::LD:
936 return RISCV::PseudoCCLD;
937 case RISCV::QC_E_LB:
938 return RISCV::PseudoCCQC_E_LB;
939 case RISCV::QC_E_LBU:
940 return RISCV::PseudoCCQC_E_LBU;
941 case RISCV::QC_E_LH:
942 return RISCV::PseudoCCQC_E_LH;
943 case RISCV::QC_E_LHU:
944 return RISCV::PseudoCCQC_E_LHU;
945 case RISCV::QC_E_LW:
946 return RISCV::PseudoCCQC_E_LW;
947 default:
948 return 0;
949 }
950}
951
952MachineInstr *RISCVInstrInfo::foldMemoryOperandImpl(
953 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
954 MachineBasicBlock::iterator InsertPt, MachineInstr &LoadMI,
955 LiveIntervals *LIS) const {
956 // For now, only handle RISCV::PseudoCCMOVGPR.
957 if (MI.getOpcode() != RISCV::PseudoCCMOVGPR)
958 return nullptr;
959
960 unsigned PredOpc = getLoadPredicatedOpcode(Opcode: LoadMI.getOpcode());
961
962 if (!STI.hasShortForwardBranchILoad() || !PredOpc)
963 return nullptr;
964
965 MachineRegisterInfo &MRI = MF.getRegInfo();
966 if (Ops.size() != 1 || (Ops[0] != 1 && Ops[0] != 2))
967 return nullptr;
968
969 bool Invert = Ops[0] == 2;
970 const MachineOperand &FalseReg = MI.getOperand(i: !Invert ? 2 : 1);
971 Register DestReg = MI.getOperand(i: 0).getReg();
972 const TargetRegisterClass *PreviousClass = MRI.getRegClass(Reg: FalseReg.getReg());
973 if (!MRI.constrainRegClass(Reg: DestReg, RC: PreviousClass))
974 return nullptr;
975
976 // Create a new predicated version of DefMI.
977 MachineInstrBuilder NewMI = BuildMI(BB&: *MI.getParent(), I: InsertPt,
978 MIMD: MI.getDebugLoc(), MCID: get(Opcode: PredOpc), DestReg);
979
980 // Copy the false register.
981 NewMI.add(MO: FalseReg);
982
983 // Copy all the DefMI operands.
984 const MCInstrDesc &DefDesc = LoadMI.getDesc();
985 for (unsigned i = 1, e = DefDesc.getNumOperands(); i != e; ++i)
986 NewMI.add(MO: LoadMI.getOperand(i));
987
988 // Add branch opcode, inverting if necessary.
989 unsigned BCC = MI.getOperand(i: MI.getNumExplicitOperands() - 3).getImm();
990 if (!Invert)
991 BCC = RISCVCC::getInverseBranchOpcode(BCC);
992 NewMI.addImm(Val: BCC);
993
994 // Copy condition portion
995 NewMI.add(MOs: {MI.getOperand(i: MI.getNumExplicitOperands() - 2),
996 MI.getOperand(i: MI.getNumExplicitOperands() - 1)});
997 NewMI.cloneMemRefs(OtherMI: LoadMI);
998 return NewMI;
999}
1000
1001void RISCVInstrInfo::movImm(MachineBasicBlock &MBB,
1002 MachineBasicBlock::iterator MBBI,
1003 const DebugLoc &DL, Register DstReg, uint64_t Val,
1004 MachineInstr::MIFlag Flag, bool DstRenamable,
1005 bool DstIsDead) const {
1006 Register SrcReg = RISCV::X0;
1007
1008 // For RV32, allow a sign or unsigned 32 bit value.
1009 if (!STI.is64Bit() && !isInt<32>(x: Val)) {
1010 // If have a uimm32 it will still fit in a register so we can allow it.
1011 if (!isUInt<32>(x: Val))
1012 report_fatal_error(reason: "Should only materialize 32-bit constants for RV32");
1013
1014 // Sign extend for generateInstSeq.
1015 Val = SignExtend64<32>(x: Val);
1016 }
1017
1018 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val, STI);
1019 assert(!Seq.empty());
1020
1021 bool SrcRenamable = false;
1022 unsigned Num = 0;
1023
1024 for (const RISCVMatInt::Inst &Inst : Seq) {
1025 bool LastItem = ++Num == Seq.size();
1026 RegState DstRegState = getDeadRegState(B: DstIsDead && LastItem) |
1027 getRenamableRegState(B: DstRenamable);
1028 RegState SrcRegState = getKillRegState(B: SrcReg != RISCV::X0) |
1029 getRenamableRegState(B: SrcRenamable);
1030 switch (Inst.getOpndKind()) {
1031 case RISCVMatInt::Imm:
1032 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: Inst.getOpcode()))
1033 .addReg(RegNo: DstReg, Flags: RegState::Define | DstRegState)
1034 .addImm(Val: Inst.getImm())
1035 .setMIFlag(Flag);
1036 break;
1037 case RISCVMatInt::RegX0:
1038 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: Inst.getOpcode()))
1039 .addReg(RegNo: DstReg, Flags: RegState::Define | DstRegState)
1040 .addReg(RegNo: SrcReg, Flags: SrcRegState)
1041 .addReg(RegNo: RISCV::X0)
1042 .setMIFlag(Flag);
1043 break;
1044 case RISCVMatInt::RegReg:
1045 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: Inst.getOpcode()))
1046 .addReg(RegNo: DstReg, Flags: RegState::Define | DstRegState)
1047 .addReg(RegNo: SrcReg, Flags: SrcRegState)
1048 .addReg(RegNo: SrcReg, Flags: SrcRegState)
1049 .setMIFlag(Flag);
1050 break;
1051 case RISCVMatInt::RegImm:
1052 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: Inst.getOpcode()))
1053 .addReg(RegNo: DstReg, Flags: RegState::Define | DstRegState)
1054 .addReg(RegNo: SrcReg, Flags: SrcRegState)
1055 .addImm(Val: Inst.getImm())
1056 .setMIFlag(Flag);
1057 break;
1058 }
1059
1060 // Only the first instruction has X0 as its source.
1061 SrcReg = DstReg;
1062 SrcRenamable = DstRenamable;
1063 }
1064}
1065
1066RISCVCC::CondCode RISCVInstrInfo::getCondFromBranchOpc(unsigned Opc) {
1067 switch (Opc) {
1068 default:
1069 return RISCVCC::COND_INVALID;
1070 case RISCV::BEQ:
1071 case RISCV::BEQI:
1072 case RISCV::CV_BEQIMM:
1073 case RISCV::QC_BEQI:
1074 case RISCV::QC_E_BEQI:
1075 case RISCV::NDS_BBC:
1076 case RISCV::NDS_BEQC:
1077 return RISCVCC::COND_EQ;
1078 case RISCV::BNE:
1079 case RISCV::BNEI:
1080 case RISCV::QC_BNEI:
1081 case RISCV::QC_E_BNEI:
1082 case RISCV::CV_BNEIMM:
1083 case RISCV::NDS_BBS:
1084 case RISCV::NDS_BNEC:
1085 return RISCVCC::COND_NE;
1086 case RISCV::BLT:
1087 case RISCV::QC_BLTI:
1088 case RISCV::QC_E_BLTI:
1089 return RISCVCC::COND_LT;
1090 case RISCV::BGE:
1091 case RISCV::QC_BGEI:
1092 case RISCV::QC_E_BGEI:
1093 return RISCVCC::COND_GE;
1094 case RISCV::BLTU:
1095 case RISCV::QC_BLTUI:
1096 case RISCV::QC_E_BLTUI:
1097 return RISCVCC::COND_LTU;
1098 case RISCV::BGEU:
1099 case RISCV::QC_BGEUI:
1100 case RISCV::QC_E_BGEUI:
1101 return RISCVCC::COND_GEU;
1102 }
1103}
1104
1105bool RISCVInstrInfo::evaluateCondBranch(RISCVCC::CondCode CC, int64_t C0,
1106 int64_t C1) {
1107 switch (CC) {
1108 default:
1109 llvm_unreachable("Unexpected CC");
1110 case RISCVCC::COND_EQ:
1111 return C0 == C1;
1112 case RISCVCC::COND_NE:
1113 return C0 != C1;
1114 case RISCVCC::COND_LT:
1115 return C0 < C1;
1116 case RISCVCC::COND_GE:
1117 return C0 >= C1;
1118 case RISCVCC::COND_LTU:
1119 return (uint64_t)C0 < (uint64_t)C1;
1120 case RISCVCC::COND_GEU:
1121 return (uint64_t)C0 >= (uint64_t)C1;
1122 }
1123}
1124
1125// The contents of values added to Cond are not examined outside of
1126// RISCVInstrInfo, giving us flexibility in what to push to it. For RISCV, we
1127// push BranchOpcode, Reg1, Reg2.
1128static void parseCondBranch(MachineInstr &LastInst, MachineBasicBlock *&Target,
1129 SmallVectorImpl<MachineOperand> &Cond) {
1130 // Block ends with fall-through condbranch.
1131 assert(LastInst.getDesc().isConditionalBranch() &&
1132 "Unknown conditional branch");
1133 Target = LastInst.getOperand(i: 2).getMBB();
1134 Cond.push_back(Elt: MachineOperand::CreateImm(Val: LastInst.getOpcode()));
1135 Cond.push_back(Elt: LastInst.getOperand(i: 0));
1136 Cond.push_back(Elt: LastInst.getOperand(i: 1));
1137}
1138
1139static unsigned getInverseXqcicmOpcode(unsigned Opcode) {
1140 switch (Opcode) {
1141 default:
1142 llvm_unreachable("Unexpected Opcode");
1143 case RISCV::QC_MVEQ:
1144 return RISCV::QC_MVNE;
1145 case RISCV::QC_MVNE:
1146 return RISCV::QC_MVEQ;
1147 case RISCV::QC_MVLT:
1148 return RISCV::QC_MVGE;
1149 case RISCV::QC_MVGE:
1150 return RISCV::QC_MVLT;
1151 case RISCV::QC_MVLTU:
1152 return RISCV::QC_MVGEU;
1153 case RISCV::QC_MVGEU:
1154 return RISCV::QC_MVLTU;
1155 case RISCV::QC_MVEQI:
1156 return RISCV::QC_MVNEI;
1157 case RISCV::QC_MVNEI:
1158 return RISCV::QC_MVEQI;
1159 case RISCV::QC_MVLTI:
1160 return RISCV::QC_MVGEI;
1161 case RISCV::QC_MVGEI:
1162 return RISCV::QC_MVLTI;
1163 case RISCV::QC_MVLTUI:
1164 return RISCV::QC_MVGEUI;
1165 case RISCV::QC_MVGEUI:
1166 return RISCV::QC_MVLTUI;
1167 }
1168}
1169
1170unsigned RISCVCC::getBrCond(RISCVCC::CondCode CC, unsigned SelectOpc) {
1171 switch (SelectOpc) {
1172 default:
1173 switch (CC) {
1174 default:
1175 llvm_unreachable("Unexpected condition code!");
1176 case RISCVCC::COND_EQ:
1177 return RISCV::BEQ;
1178 case RISCVCC::COND_NE:
1179 return RISCV::BNE;
1180 case RISCVCC::COND_LT:
1181 return RISCV::BLT;
1182 case RISCVCC::COND_GE:
1183 return RISCV::BGE;
1184 case RISCVCC::COND_LTU:
1185 return RISCV::BLTU;
1186 case RISCVCC::COND_GEU:
1187 return RISCV::BGEU;
1188 }
1189 break;
1190 case RISCV::Select_GPR_Using_CC_Imm5_Zibi:
1191 switch (CC) {
1192 default:
1193 llvm_unreachable("Unexpected condition code!");
1194 case RISCVCC::COND_EQ:
1195 return RISCV::BEQI;
1196 case RISCVCC::COND_NE:
1197 return RISCV::BNEI;
1198 }
1199 break;
1200 case RISCV::Select_GPR_Using_CC_SImm5_CV:
1201 switch (CC) {
1202 default:
1203 llvm_unreachable("Unexpected condition code!");
1204 case RISCVCC::COND_EQ:
1205 return RISCV::CV_BEQIMM;
1206 case RISCVCC::COND_NE:
1207 return RISCV::CV_BNEIMM;
1208 }
1209 break;
1210 case RISCV::Select_GPRNoX0_Using_CC_SImm5NonZero_QC:
1211 switch (CC) {
1212 default:
1213 llvm_unreachable("Unexpected condition code!");
1214 case RISCVCC::COND_EQ:
1215 return RISCV::QC_BEQI;
1216 case RISCVCC::COND_NE:
1217 return RISCV::QC_BNEI;
1218 case RISCVCC::COND_LT:
1219 return RISCV::QC_BLTI;
1220 case RISCVCC::COND_GE:
1221 return RISCV::QC_BGEI;
1222 }
1223 break;
1224 case RISCV::Select_GPRNoX0_Using_CC_UImm5NonZero_QC:
1225 switch (CC) {
1226 default:
1227 llvm_unreachable("Unexpected condition code!");
1228 case RISCVCC::COND_LTU:
1229 return RISCV::QC_BLTUI;
1230 case RISCVCC::COND_GEU:
1231 return RISCV::QC_BGEUI;
1232 }
1233 break;
1234 case RISCV::Select_GPRNoX0_Using_CC_SImm16NonZero_QC:
1235 switch (CC) {
1236 default:
1237 llvm_unreachable("Unexpected condition code!");
1238 case RISCVCC::COND_EQ:
1239 return RISCV::QC_E_BEQI;
1240 case RISCVCC::COND_NE:
1241 return RISCV::QC_E_BNEI;
1242 case RISCVCC::COND_LT:
1243 return RISCV::QC_E_BLTI;
1244 case RISCVCC::COND_GE:
1245 return RISCV::QC_E_BGEI;
1246 }
1247 break;
1248 case RISCV::Select_GPRNoX0_Using_CC_UImm16NonZero_QC:
1249 switch (CC) {
1250 default:
1251 llvm_unreachable("Unexpected condition code!");
1252 case RISCVCC::COND_LTU:
1253 return RISCV::QC_E_BLTUI;
1254 case RISCVCC::COND_GEU:
1255 return RISCV::QC_E_BGEUI;
1256 }
1257 break;
1258 case RISCV::Select_GPR_Using_CC_UImmLog2XLen_NDS:
1259 switch (CC) {
1260 default:
1261 llvm_unreachable("Unexpected condition code!");
1262 case RISCVCC::COND_EQ:
1263 return RISCV::NDS_BBC;
1264 case RISCVCC::COND_NE:
1265 return RISCV::NDS_BBS;
1266 }
1267 break;
1268 case RISCV::Select_GPR_Using_CC_UImm7_NDS:
1269 switch (CC) {
1270 default:
1271 llvm_unreachable("Unexpected condition code!");
1272 case RISCVCC::COND_EQ:
1273 return RISCV::NDS_BEQC;
1274 case RISCVCC::COND_NE:
1275 return RISCV::NDS_BNEC;
1276 }
1277 break;
1278 }
1279}
1280
1281RISCVCC::CondCode RISCVCC::getInverseBranchCondition(RISCVCC::CondCode CC) {
1282 switch (CC) {
1283 default:
1284 llvm_unreachable("Unrecognized conditional branch");
1285 case RISCVCC::COND_EQ:
1286 return RISCVCC::COND_NE;
1287 case RISCVCC::COND_NE:
1288 return RISCVCC::COND_EQ;
1289 case RISCVCC::COND_LT:
1290 return RISCVCC::COND_GE;
1291 case RISCVCC::COND_GE:
1292 return RISCVCC::COND_LT;
1293 case RISCVCC::COND_LTU:
1294 return RISCVCC::COND_GEU;
1295 case RISCVCC::COND_GEU:
1296 return RISCVCC::COND_LTU;
1297 }
1298}
1299
1300// Return inverse branch
1301unsigned RISCVCC::getInverseBranchOpcode(unsigned BCC) {
1302 switch (BCC) {
1303 default:
1304 llvm_unreachable("Unexpected branch opcode!");
1305 case RISCV::BEQ:
1306 return RISCV::BNE;
1307 case RISCV::BEQI:
1308 return RISCV::BNEI;
1309 case RISCV::BNE:
1310 return RISCV::BEQ;
1311 case RISCV::BNEI:
1312 return RISCV::BEQI;
1313 case RISCV::BLT:
1314 return RISCV::BGE;
1315 case RISCV::BGE:
1316 return RISCV::BLT;
1317 case RISCV::BLTU:
1318 return RISCV::BGEU;
1319 case RISCV::BGEU:
1320 return RISCV::BLTU;
1321 case RISCV::CV_BEQIMM:
1322 return RISCV::CV_BNEIMM;
1323 case RISCV::CV_BNEIMM:
1324 return RISCV::CV_BEQIMM;
1325 case RISCV::QC_BEQI:
1326 return RISCV::QC_BNEI;
1327 case RISCV::QC_BNEI:
1328 return RISCV::QC_BEQI;
1329 case RISCV::QC_BLTI:
1330 return RISCV::QC_BGEI;
1331 case RISCV::QC_BGEI:
1332 return RISCV::QC_BLTI;
1333 case RISCV::QC_BLTUI:
1334 return RISCV::QC_BGEUI;
1335 case RISCV::QC_BGEUI:
1336 return RISCV::QC_BLTUI;
1337 case RISCV::QC_E_BEQI:
1338 return RISCV::QC_E_BNEI;
1339 case RISCV::QC_E_BNEI:
1340 return RISCV::QC_E_BEQI;
1341 case RISCV::QC_E_BLTI:
1342 return RISCV::QC_E_BGEI;
1343 case RISCV::QC_E_BGEI:
1344 return RISCV::QC_E_BLTI;
1345 case RISCV::QC_E_BLTUI:
1346 return RISCV::QC_E_BGEUI;
1347 case RISCV::QC_E_BGEUI:
1348 return RISCV::QC_E_BLTUI;
1349 case RISCV::NDS_BBC:
1350 return RISCV::NDS_BBS;
1351 case RISCV::NDS_BBS:
1352 return RISCV::NDS_BBC;
1353 case RISCV::NDS_BEQC:
1354 return RISCV::NDS_BNEC;
1355 case RISCV::NDS_BNEC:
1356 return RISCV::NDS_BEQC;
1357 }
1358}
1359
1360bool RISCVInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
1361 MachineBasicBlock *&TBB,
1362 MachineBasicBlock *&FBB,
1363 SmallVectorImpl<MachineOperand> &Cond,
1364 bool AllowModify) const {
1365 TBB = FBB = nullptr;
1366 Cond.clear();
1367
1368 // If the block has no terminators, it just falls into the block after it.
1369 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
1370 if (I == MBB.end() || !isUnpredicatedTerminator(MI: *I))
1371 return false;
1372
1373 // Count the number of terminators and find the first unconditional or
1374 // indirect branch.
1375 MachineBasicBlock::iterator FirstUncondOrIndirectBr = MBB.end();
1376 int NumTerminators = 0;
1377 for (auto J = I.getReverse(); J != MBB.rend() && isUnpredicatedTerminator(MI: *J);
1378 J++) {
1379 NumTerminators++;
1380 if (J->getDesc().isUnconditionalBranch() ||
1381 J->getDesc().isIndirectBranch()) {
1382 FirstUncondOrIndirectBr = J.getReverse();
1383 }
1384 }
1385
1386 // If AllowModify is true, we can erase any terminators after
1387 // FirstUncondOrIndirectBR.
1388 if (AllowModify && FirstUncondOrIndirectBr != MBB.end()) {
1389 while (std::next(x: FirstUncondOrIndirectBr) != MBB.end()) {
1390 std::next(x: FirstUncondOrIndirectBr)->eraseFromParent();
1391 NumTerminators--;
1392 }
1393 I = FirstUncondOrIndirectBr;
1394 }
1395
1396 // We can't handle blocks that end in an indirect branch.
1397 if (I->getDesc().isIndirectBranch())
1398 return true;
1399
1400 // We can't handle Generic branch opcodes from Global ISel.
1401 if (I->isPreISelOpcode())
1402 return true;
1403
1404 // We can't handle blocks with more than 2 terminators.
1405 if (NumTerminators > 2)
1406 return true;
1407
1408 // Handle a single unconditional branch.
1409 if (NumTerminators == 1 && I->getDesc().isUnconditionalBranch()) {
1410 TBB = getBranchDestBlock(MI: *I);
1411 return false;
1412 }
1413
1414 // Handle a single conditional branch.
1415 if (NumTerminators == 1 && I->getDesc().isConditionalBranch()) {
1416 parseCondBranch(LastInst&: *I, Target&: TBB, Cond);
1417 return false;
1418 }
1419
1420 // Handle a conditional branch followed by an unconditional branch.
1421 if (NumTerminators == 2 && std::prev(x: I)->getDesc().isConditionalBranch() &&
1422 I->getDesc().isUnconditionalBranch()) {
1423 parseCondBranch(LastInst&: *std::prev(x: I), Target&: TBB, Cond);
1424 FBB = getBranchDestBlock(MI: *I);
1425 return false;
1426 }
1427
1428 // Otherwise, we can't handle this.
1429 return true;
1430}
1431
1432unsigned RISCVInstrInfo::removeBranch(MachineBasicBlock &MBB,
1433 int *BytesRemoved) const {
1434 if (BytesRemoved)
1435 *BytesRemoved = 0;
1436 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
1437 if (I == MBB.end())
1438 return 0;
1439
1440 if (!I->getDesc().isUnconditionalBranch() &&
1441 !I->getDesc().isConditionalBranch())
1442 return 0;
1443
1444 // Remove the branch.
1445 if (BytesRemoved)
1446 *BytesRemoved += getInstSizeInBytes(MI: *I);
1447 I->eraseFromParent();
1448
1449 I = MBB.end();
1450
1451 if (I == MBB.begin())
1452 return 1;
1453 --I;
1454 if (!I->getDesc().isConditionalBranch())
1455 return 1;
1456
1457 // Remove the branch.
1458 if (BytesRemoved)
1459 *BytesRemoved += getInstSizeInBytes(MI: *I);
1460 I->eraseFromParent();
1461 return 2;
1462}
1463
1464// Inserts a branch into the end of the specific MachineBasicBlock, returning
1465// the number of instructions inserted.
1466unsigned RISCVInstrInfo::insertBranch(
1467 MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
1468 ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const {
1469 if (BytesAdded)
1470 *BytesAdded = 0;
1471
1472 // Shouldn't be a fall through.
1473 assert(TBB && "insertBranch must not be told to insert a fallthrough");
1474 assert((Cond.size() == 3 || Cond.size() == 0) &&
1475 "RISC-V branch conditions have two components!");
1476
1477 // Unconditional branch.
1478 if (Cond.empty()) {
1479 MachineInstr &MI = *BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: RISCV::PseudoBR)).addMBB(MBB: TBB);
1480 if (BytesAdded)
1481 *BytesAdded += getInstSizeInBytes(MI);
1482 return 1;
1483 }
1484
1485 // Either a one or two-way conditional branch.
1486 MachineInstr &CondMI = *BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: Cond[0].getImm()))
1487 .add(MO: Cond[1])
1488 .add(MO: Cond[2])
1489 .addMBB(MBB: TBB);
1490 if (BytesAdded)
1491 *BytesAdded += getInstSizeInBytes(MI: CondMI);
1492
1493 // One-way conditional branch.
1494 if (!FBB)
1495 return 1;
1496
1497 // Two-way conditional branch.
1498 MachineInstr &MI = *BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: RISCV::PseudoBR)).addMBB(MBB: FBB);
1499 if (BytesAdded)
1500 *BytesAdded += getInstSizeInBytes(MI);
1501 return 2;
1502}
1503
1504void RISCVInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
1505 MachineBasicBlock &DestBB,
1506 MachineBasicBlock &RestoreBB,
1507 const DebugLoc &DL, int64_t BrOffset,
1508 RegScavenger *RS) const {
1509 assert(RS && "RegScavenger required for long branching");
1510 assert(MBB.empty() &&
1511 "new block should be inserted for expanding unconditional branch");
1512 assert(MBB.pred_size() == 1);
1513 assert(RestoreBB.empty() &&
1514 "restore block should be inserted for restoring clobbered registers");
1515
1516 MachineFunction *MF = MBB.getParent();
1517 MachineRegisterInfo &MRI = MF->getRegInfo();
1518 RISCVMachineFunctionInfo *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
1519 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
1520
1521 if (!isInt<32>(x: BrOffset))
1522 report_fatal_error(
1523 reason: "Branch offsets outside of the signed 32-bit range not supported");
1524
1525 // FIXME: A virtual register must be used initially, as the register
1526 // scavenger won't work with empty blocks (SIInstrInfo::insertIndirectBranch
1527 // uses the same workaround).
1528 Register ScratchReg = MRI.createVirtualRegister(RegClass: &RISCV::GPRJALRRegClass);
1529 auto II = MBB.end();
1530 // We may also update the jump target to RestoreBB later.
1531 MachineInstr &MI = *BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::PseudoJump))
1532 .addReg(RegNo: ScratchReg, Flags: RegState::Define | RegState::Dead)
1533 .addMBB(MBB: &DestBB, TargetFlags: RISCVII::MO_CALL);
1534
1535 RS->enterBasicBlockEnd(MBB);
1536 const TargetRegisterClass *RC = &RISCV::GPRRegClass;
1537 if (STI.hasStdExtZicfilp())
1538 RC = &RISCV::GPRX7RegClass;
1539 Register TmpGPR =
1540 RS->scavengeRegisterBackwards(RC: *RC, To: MI.getIterator(),
1541 /*RestoreAfter=*/false, /*SpAdj=*/SPAdj: 0,
1542 /*AllowSpill=*/false);
1543 if (TmpGPR.isValid())
1544 RS->setRegUsed(Reg: TmpGPR);
1545 else {
1546 // The case when there is no scavenged register needs special handling.
1547
1548 // Pick s11(or s1 for rve) because it doesn't make a difference.
1549 TmpGPR = STI.hasStdExtE() ? RISCV::X9 : RISCV::X27;
1550 // Force t2 if Zicfilp is on
1551 if (STI.hasStdExtZicfilp())
1552 TmpGPR = RISCV::X7;
1553
1554 int FrameIndex = RVFI->getBranchRelaxationScratchFrameIndex();
1555 if (FrameIndex == -1)
1556 report_fatal_error(reason: "underestimated function size");
1557
1558 storeRegToStackSlot(MBB, I: MI, SrcReg: TmpGPR, /*IsKill=*/true, FI: FrameIndex,
1559 RC: &RISCV::GPRRegClass, VReg: Register());
1560 TRI->eliminateFrameIndex(MI: std::prev(x: MI.getIterator()),
1561 /*SpAdj=*/SPAdj: 0, /*FIOperandNum=*/1);
1562
1563 MI.getOperand(i: 1).setMBB(&RestoreBB);
1564
1565 loadRegFromStackSlot(MBB&: RestoreBB, I: RestoreBB.end(), DstReg: TmpGPR, FI: FrameIndex,
1566 RC: &RISCV::GPRRegClass, VReg: Register());
1567 TRI->eliminateFrameIndex(MI: RestoreBB.back(),
1568 /*SpAdj=*/SPAdj: 0, /*FIOperandNum=*/1);
1569 }
1570
1571 MRI.replaceRegWith(FromReg: ScratchReg, ToReg: TmpGPR);
1572 MRI.clearVirtRegs();
1573}
1574
1575bool RISCVInstrInfo::reverseBranchCondition(
1576 SmallVectorImpl<MachineOperand> &Cond) const {
1577 assert((Cond.size() == 3) && "Invalid branch condition!");
1578
1579 Cond[0].setImm(RISCVCC::getInverseBranchOpcode(BCC: Cond[0].getImm()));
1580
1581 return false;
1582}
1583
1584// Return true if the instruction is a load immediate instruction (i.e.
1585// (ADDI x0, imm) or (BSETI x0, imm)).
1586static bool isLoadImm(const MachineInstr *MI, int64_t &Imm) {
1587 if (MI->getOpcode() == RISCV::ADDI && MI->getOperand(i: 1).isReg() &&
1588 MI->getOperand(i: 1).getReg() == RISCV::X0) {
1589 Imm = MI->getOperand(i: 2).getImm();
1590 return true;
1591 }
1592 // BSETI can be used to create power of 2 constants. Only 2048 is currently
1593 // interesting because it is 1 more than the maximum ADDI constant.
1594 if (MI->getOpcode() == RISCV::BSETI && MI->getOperand(i: 1).isReg() &&
1595 MI->getOperand(i: 1).getReg() == RISCV::X0 &&
1596 MI->getOperand(i: 2).getImm() == 11) {
1597 Imm = 2048;
1598 return true;
1599 }
1600 return false;
1601}
1602
1603bool RISCVInstrInfo::isFromLoadImm(const MachineRegisterInfo &MRI,
1604 const MachineOperand &Op, int64_t &Imm) {
1605 // Either a load from immediate instruction or X0.
1606 if (!Op.isReg())
1607 return false;
1608
1609 Register Reg = Op.getReg();
1610 if (Reg == RISCV::X0) {
1611 Imm = 0;
1612 return true;
1613 }
1614 return Reg.isVirtual() && isLoadImm(MI: MRI.getVRegDef(Reg), Imm);
1615}
1616
1617bool RISCVInstrInfo::optimizeCondBranch(MachineInstr &MI) const {
1618 bool IsSigned = false;
1619 bool IsEquality = false;
1620 switch (MI.getOpcode()) {
1621 default:
1622 return false;
1623 case RISCV::BEQ:
1624 case RISCV::BNE:
1625 IsEquality = true;
1626 break;
1627 case RISCV::BGE:
1628 case RISCV::BLT:
1629 IsSigned = true;
1630 break;
1631 case RISCV::BGEU:
1632 case RISCV::BLTU:
1633 break;
1634 }
1635
1636 MachineBasicBlock *MBB = MI.getParent();
1637 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
1638
1639 const MachineOperand &LHS = MI.getOperand(i: 0);
1640 const MachineOperand &RHS = MI.getOperand(i: 1);
1641 MachineBasicBlock *TBB = MI.getOperand(i: 2).getMBB();
1642
1643 RISCVCC::CondCode CC = getCondFromBranchOpc(Opc: MI.getOpcode());
1644 assert(CC != RISCVCC::COND_INVALID);
1645
1646 // Canonicalize conditional branches which can be constant folded into
1647 // beqz or bnez. We can't modify the CFG here.
1648 int64_t C0, C1;
1649 if (isFromLoadImm(MRI, Op: LHS, Imm&: C0) && isFromLoadImm(MRI, Op: RHS, Imm&: C1)) {
1650 unsigned NewOpc = evaluateCondBranch(CC, C0, C1) ? RISCV::BEQ : RISCV::BNE;
1651 // Build the new branch and remove the old one.
1652 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
1653 .addReg(RegNo: RISCV::X0)
1654 .addReg(RegNo: RISCV::X0)
1655 .addMBB(MBB: TBB);
1656 MI.eraseFromParent();
1657 return true;
1658 }
1659
1660 if (IsEquality)
1661 return false;
1662
1663 // For two constants C0 and C1 from
1664 // ```
1665 // li Y, C0
1666 // li Z, C1
1667 // ```
1668 // 1. if C1 = C0 + 1
1669 // we can turn:
1670 // (a) blt Y, X -> bge X, Z
1671 // (b) bge Y, X -> blt X, Z
1672 //
1673 // 2. if C1 = C0 - 1
1674 // we can turn:
1675 // (a) blt X, Y -> bge Z, X
1676 // (b) bge X, Y -> blt Z, X
1677 //
1678 // To make sure this optimization is really beneficial, we only
1679 // optimize for cases where Y had only one use (i.e. only used by the branch).
1680 // Try to find the register for constant Z; return
1681 // invalid register otherwise.
1682 auto searchConst = [&](int64_t C1) -> Register {
1683 MachineBasicBlock::reverse_iterator II(&MI), E = MBB->rend();
1684 auto DefC1 = std::find_if(first: ++II, last: E, pred: [&](const MachineInstr &I) -> bool {
1685 int64_t Imm;
1686 return isLoadImm(MI: &I, Imm) && Imm == C1 &&
1687 I.getOperand(i: 0).getReg().isVirtual();
1688 });
1689 if (DefC1 != E)
1690 return DefC1->getOperand(i: 0).getReg();
1691
1692 return Register();
1693 };
1694
1695 unsigned NewOpc = RISCVCC::getBrCond(CC: getInverseBranchCondition(CC));
1696
1697 // Might be case 1.
1698 // Don't change 0 to 1 since we can use x0.
1699 // For unsigned cases changing -1U to 0 would be incorrect.
1700 // The incorrect case for signed would be INT_MAX, but isFromLoadImm can't
1701 // return that.
1702 if (isFromLoadImm(MRI, Op: LHS, Imm&: C0) && C0 != 0 && LHS.getReg().isVirtual() &&
1703 MRI.hasOneUse(RegNo: LHS.getReg()) && (IsSigned || C0 != -1)) {
1704 assert((isInt<12>(C0) || C0 == 2048) && "Unexpected immediate");
1705 if (Register RegZ = searchConst(C0 + 1)) {
1706 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
1707 .add(MO: RHS)
1708 .addReg(RegNo: RegZ)
1709 .addMBB(MBB: TBB);
1710 // We might extend the live range of Z, clear its kill flag to
1711 // account for this.
1712 MRI.clearKillFlags(Reg: RegZ);
1713 MI.eraseFromParent();
1714 return true;
1715 }
1716 }
1717
1718 // Might be case 2.
1719 // For signed cases we don't want to change 0 since we can use x0.
1720 // For unsigned cases changing 0 to -1U would be incorrect.
1721 // The incorrect case for signed would be INT_MIN, but isFromLoadImm can't
1722 // return that.
1723 if (isFromLoadImm(MRI, Op: RHS, Imm&: C0) && C0 != 0 && RHS.getReg().isVirtual() &&
1724 MRI.hasOneUse(RegNo: RHS.getReg())) {
1725 assert((isInt<12>(C0) || C0 == 2048) && "Unexpected immediate");
1726 if (Register RegZ = searchConst(C0 - 1)) {
1727 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
1728 .addReg(RegNo: RegZ)
1729 .add(MO: LHS)
1730 .addMBB(MBB: TBB);
1731 // We might extend the live range of Z, clear its kill flag to
1732 // account for this.
1733 MRI.clearKillFlags(Reg: RegZ);
1734 MI.eraseFromParent();
1735 return true;
1736 }
1737 }
1738
1739 return false;
1740}
1741
1742MachineBasicBlock *
1743RISCVInstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
1744 assert(MI.getDesc().isBranch() && "Unexpected opcode!");
1745 // The branch target is always the last operand.
1746 int NumOp = MI.getNumExplicitOperands();
1747 return MI.getOperand(i: NumOp - 1).getMBB();
1748}
1749
1750bool RISCVInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
1751 int64_t BrOffset) const {
1752 unsigned XLen = STI.getXLen();
1753 // Ideally we could determine the supported branch offset from the
1754 // RISCVII::FormMask, but this can't be used for Pseudo instructions like
1755 // PseudoBR.
1756 switch (BranchOp) {
1757 default:
1758 llvm_unreachable("Unexpected opcode!");
1759 case RISCV::NDS_BBC:
1760 case RISCV::NDS_BBS:
1761 case RISCV::NDS_BEQC:
1762 case RISCV::NDS_BNEC:
1763 return isInt<11>(x: BrOffset);
1764 case RISCV::BEQ:
1765 case RISCV::BNE:
1766 case RISCV::BLT:
1767 case RISCV::BGE:
1768 case RISCV::BLTU:
1769 case RISCV::BGEU:
1770 case RISCV::BEQI:
1771 case RISCV::BNEI:
1772 case RISCV::CV_BEQIMM:
1773 case RISCV::CV_BNEIMM:
1774 case RISCV::QC_BEQI:
1775 case RISCV::QC_BNEI:
1776 case RISCV::QC_BGEI:
1777 case RISCV::QC_BLTI:
1778 case RISCV::QC_BLTUI:
1779 case RISCV::QC_BGEUI:
1780 case RISCV::QC_E_BEQI:
1781 case RISCV::QC_E_BNEI:
1782 case RISCV::QC_E_BGEI:
1783 case RISCV::QC_E_BLTI:
1784 case RISCV::QC_E_BLTUI:
1785 case RISCV::QC_E_BGEUI:
1786 return isInt<13>(x: BrOffset);
1787 case RISCV::JAL:
1788 case RISCV::PseudoBR:
1789 return isInt<21>(x: BrOffset);
1790 case RISCV::PseudoJump:
1791 return isInt<32>(x: SignExtend64(X: BrOffset + 0x800, B: XLen));
1792 }
1793}
1794
1795// If the operation has a predicated pseudo instruction, return the pseudo
1796// instruction opcode. Otherwise, return RISCV::INSTRUCTION_LIST_END.
1797// TODO: Support more operations.
1798unsigned getPredicatedOpcode(unsigned Opcode) {
1799 // clang-format off
1800 switch (Opcode) {
1801 case RISCV::ADD: return RISCV::PseudoCCADD;
1802 case RISCV::SUB: return RISCV::PseudoCCSUB;
1803 case RISCV::SLL: return RISCV::PseudoCCSLL;
1804 case RISCV::SRL: return RISCV::PseudoCCSRL;
1805 case RISCV::SRA: return RISCV::PseudoCCSRA;
1806 case RISCV::AND: return RISCV::PseudoCCAND;
1807 case RISCV::OR: return RISCV::PseudoCCOR;
1808 case RISCV::XOR: return RISCV::PseudoCCXOR;
1809 case RISCV::MAX: return RISCV::PseudoCCMAX;
1810 case RISCV::MAXU: return RISCV::PseudoCCMAXU;
1811 case RISCV::MIN: return RISCV::PseudoCCMIN;
1812 case RISCV::MINU: return RISCV::PseudoCCMINU;
1813 case RISCV::MUL: return RISCV::PseudoCCMUL;
1814 case RISCV::LUI: return RISCV::PseudoCCLUI;
1815 case RISCV::QC_LI: return RISCV::PseudoCCQC_LI;
1816 case RISCV::QC_E_LI: return RISCV::PseudoCCQC_E_LI;
1817
1818 case RISCV::ADDI: return RISCV::PseudoCCADDI;
1819 case RISCV::SLLI: return RISCV::PseudoCCSLLI;
1820 case RISCV::SRLI: return RISCV::PseudoCCSRLI;
1821 case RISCV::SRAI: return RISCV::PseudoCCSRAI;
1822 case RISCV::ANDI: return RISCV::PseudoCCANDI;
1823 case RISCV::ORI: return RISCV::PseudoCCORI;
1824 case RISCV::XORI: return RISCV::PseudoCCXORI;
1825
1826 case RISCV::ADDW: return RISCV::PseudoCCADDW;
1827 case RISCV::SUBW: return RISCV::PseudoCCSUBW;
1828 case RISCV::SLLW: return RISCV::PseudoCCSLLW;
1829 case RISCV::SRLW: return RISCV::PseudoCCSRLW;
1830 case RISCV::SRAW: return RISCV::PseudoCCSRAW;
1831
1832 case RISCV::ADDIW: return RISCV::PseudoCCADDIW;
1833 case RISCV::SLLIW: return RISCV::PseudoCCSLLIW;
1834 case RISCV::SRLIW: return RISCV::PseudoCCSRLIW;
1835 case RISCV::SRAIW: return RISCV::PseudoCCSRAIW;
1836
1837 case RISCV::ANDN: return RISCV::PseudoCCANDN;
1838 case RISCV::ORN: return RISCV::PseudoCCORN;
1839 case RISCV::XNOR: return RISCV::PseudoCCXNOR;
1840
1841 case RISCV::NDS_BFOS: return RISCV::PseudoCCNDS_BFOS;
1842 case RISCV::NDS_BFOZ: return RISCV::PseudoCCNDS_BFOZ;
1843 }
1844 // clang-format on
1845
1846 return RISCV::INSTRUCTION_LIST_END;
1847}
1848
1849/// Identify instructions that can be folded into a CCMOV instruction, and
1850/// return the defining instruction.
1851static MachineInstr *canFoldAsPredicatedOp(Register Reg,
1852 const MachineRegisterInfo &MRI,
1853 const TargetInstrInfo *TII,
1854 const RISCVSubtarget &STI) {
1855 if (!Reg.isVirtual())
1856 return nullptr;
1857 if (!MRI.hasOneNonDBGUse(RegNo: Reg))
1858 return nullptr;
1859 MachineInstr *MI = MRI.getVRegDef(Reg);
1860 if (!MI)
1861 return nullptr;
1862
1863 if (!STI.hasShortForwardBranchIMinMax() &&
1864 (MI->getOpcode() == RISCV::MAX || MI->getOpcode() == RISCV::MIN ||
1865 MI->getOpcode() == RISCV::MINU || MI->getOpcode() == RISCV::MAXU))
1866 return nullptr;
1867
1868 if (!STI.hasShortForwardBranchIMul() && MI->getOpcode() == RISCV::MUL)
1869 return nullptr;
1870
1871 // Check if MI can be predicated and folded into the CCMOV.
1872 if (getPredicatedOpcode(Opcode: MI->getOpcode()) == RISCV::INSTRUCTION_LIST_END)
1873 return nullptr;
1874 // Don't predicate li idiom.
1875 if (MI->getOpcode() == RISCV::ADDI && MI->getOperand(i: 1).isReg() &&
1876 MI->getOperand(i: 1).getReg() == RISCV::X0)
1877 return nullptr;
1878 // Check if MI has any other defs or physreg uses.
1879 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands())) {
1880 // Reject frame index operands, PEI can't handle the predicated pseudos.
1881 if (MO.isFI() || MO.isCPI() || MO.isJTI())
1882 return nullptr;
1883 if (!MO.isReg())
1884 continue;
1885 // MI can't have any tied operands, that would conflict with predication.
1886 if (MO.isTied())
1887 return nullptr;
1888 if (MO.isDef())
1889 return nullptr;
1890 // Allow constant physregs.
1891 if (MO.getReg().isPhysical() && !MRI.isConstantPhysReg(PhysReg: MO.getReg()))
1892 return nullptr;
1893 }
1894 bool DontMoveAcrossStores = true;
1895 if (!MI->isSafeToMove(SawStore&: DontMoveAcrossStores))
1896 return nullptr;
1897 return MI;
1898}
1899
1900MachineInstr *
1901RISCVInstrInfo::optimizeSelect(MachineInstr &MI,
1902 SmallPtrSetImpl<MachineInstr *> &SeenMIs,
1903 bool PreferFalse) const {
1904 assert(MI.getOpcode() == RISCV::PseudoCCMOVGPR &&
1905 "Unknown select instruction");
1906 if (!STI.hasShortForwardBranchIALU())
1907 return nullptr;
1908
1909 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
1910 MachineInstr *DefMI =
1911 canFoldAsPredicatedOp(Reg: MI.getOperand(i: 2).getReg(), MRI, TII: this, STI);
1912 bool Invert = !DefMI;
1913 if (!DefMI)
1914 DefMI = canFoldAsPredicatedOp(Reg: MI.getOperand(i: 1).getReg(), MRI, TII: this, STI);
1915 if (!DefMI)
1916 return nullptr;
1917
1918 // Find new register class to use.
1919 MachineOperand FalseReg = MI.getOperand(i: Invert ? 2 : 1);
1920 Register DestReg = MI.getOperand(i: 0).getReg();
1921 const TargetRegisterClass *PreviousClass = MRI.getRegClass(Reg: FalseReg.getReg());
1922 if (!MRI.constrainRegClass(Reg: DestReg, RC: PreviousClass))
1923 return nullptr;
1924
1925 unsigned PredOpc = getPredicatedOpcode(Opcode: DefMI->getOpcode());
1926 assert(PredOpc != RISCV::INSTRUCTION_LIST_END && "Unexpected opcode!");
1927
1928 // Create a new predicated version of DefMI.
1929 MachineInstrBuilder NewMI =
1930 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: PredOpc), DestReg);
1931
1932 // Copy the false register.
1933 NewMI.add(MO: FalseReg);
1934
1935 // Copy all the DefMI operands.
1936 const MCInstrDesc &DefDesc = DefMI->getDesc();
1937 for (unsigned i = 1, e = DefDesc.getNumOperands(); i != e; ++i)
1938 NewMI.add(MO: DefMI->getOperand(i));
1939
1940 // Add branch opcode, inverting if necessary.
1941 unsigned BCCOpcode = MI.getOperand(i: MI.getNumExplicitOperands() - 3).getImm();
1942 if (Invert)
1943 BCCOpcode = RISCVCC::getInverseBranchOpcode(BCC: BCCOpcode);
1944 NewMI.addImm(Val: BCCOpcode);
1945
1946 // Copy the condition portion.
1947 NewMI.add(MO: MI.getOperand(i: MI.getNumExplicitOperands() - 2));
1948 NewMI.add(MO: MI.getOperand(i: MI.getNumExplicitOperands() - 1));
1949
1950 // Update SeenMIs set: register newly created MI and erase removed DefMI.
1951 SeenMIs.insert(Ptr: NewMI);
1952 SeenMIs.erase(Ptr: DefMI);
1953
1954 // If MI is inside a loop, and DefMI is outside the loop, then kill flags on
1955 // DefMI would be invalid when transferred inside the loop. Checking for a
1956 // loop is expensive, but at least remove kill flags if they are in different
1957 // BBs.
1958 if (DefMI->getParent() != MI.getParent())
1959 NewMI->clearKillInfo();
1960
1961 // The caller will erase MI, but not DefMI.
1962 DefMI->eraseFromParent();
1963 return NewMI;
1964}
1965
1966unsigned RISCVInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
1967 if (MI.isMetaInstruction())
1968 return 0;
1969
1970 unsigned Opcode = MI.getOpcode();
1971
1972 if (Opcode == TargetOpcode::INLINEASM ||
1973 Opcode == TargetOpcode::INLINEASM_BR) {
1974 const MachineFunction &MF = *MI.getParent()->getParent();
1975 return getInlineAsmLength(Str: MI.getOperand(i: 0).getSymbolName(),
1976 MAI: *MF.getTarget().getMCAsmInfo());
1977 }
1978
1979 if (requiresNTLHint(MI)) {
1980 if (STI.hasStdExtZca()) {
1981 if (isCompressibleInst(MI, STI))
1982 return 4; // c.ntl.all + c.load/c.store
1983 return 6; // c.ntl.all + load/store
1984 }
1985 return 8; // ntl.all + load/store
1986 }
1987
1988 if (Opcode == TargetOpcode::BUNDLE)
1989 return getInstBundleLength(MI);
1990
1991 if (MI.getParent() && MI.getParent()->getParent()) {
1992 if (isCompressibleInst(MI, STI))
1993 return 2;
1994 }
1995
1996 switch (Opcode) {
1997 case RISCV::PseudoMV_FPR16INX:
1998 case RISCV::PseudoMV_FPR32INX:
1999 // MV is always compressible to either c.mv or c.li rd, 0.
2000 return STI.hasStdExtZca() ? 2 : 4;
2001 // Below cases are for short forward branch pseudos
2002 case RISCV::PseudoCCMOVGPRNoX0:
2003 return get(Opcode: MI.getOperand(i: MI.getNumExplicitOperands() - 3).getImm())
2004 .getSize() +
2005 2;
2006 case RISCV::PseudoCCMOVGPR:
2007 case RISCV::PseudoCCADD:
2008 case RISCV::PseudoCCSUB:
2009 case RISCV::PseudoCCSLL:
2010 case RISCV::PseudoCCSRL:
2011 case RISCV::PseudoCCSRA:
2012 case RISCV::PseudoCCAND:
2013 case RISCV::PseudoCCOR:
2014 case RISCV::PseudoCCXOR:
2015 case RISCV::PseudoCCADDI:
2016 case RISCV::PseudoCCANDI:
2017 case RISCV::PseudoCCORI:
2018 case RISCV::PseudoCCXORI:
2019 case RISCV::PseudoCCLUI:
2020 case RISCV::PseudoCCSLLI:
2021 case RISCV::PseudoCCSRLI:
2022 case RISCV::PseudoCCSRAI:
2023 case RISCV::PseudoCCADDW:
2024 case RISCV::PseudoCCSUBW:
2025 case RISCV::PseudoCCSLLW:
2026 case RISCV::PseudoCCSRLW:
2027 case RISCV::PseudoCCSRAW:
2028 case RISCV::PseudoCCADDIW:
2029 case RISCV::PseudoCCSLLIW:
2030 case RISCV::PseudoCCSRLIW:
2031 case RISCV::PseudoCCSRAIW:
2032 case RISCV::PseudoCCANDN:
2033 case RISCV::PseudoCCORN:
2034 case RISCV::PseudoCCXNOR:
2035 case RISCV::PseudoCCMAX:
2036 case RISCV::PseudoCCMIN:
2037 case RISCV::PseudoCCMAXU:
2038 case RISCV::PseudoCCMINU:
2039 case RISCV::PseudoCCMUL:
2040 case RISCV::PseudoCCLB:
2041 case RISCV::PseudoCCLH:
2042 case RISCV::PseudoCCLW:
2043 case RISCV::PseudoCCLHU:
2044 case RISCV::PseudoCCLBU:
2045 case RISCV::PseudoCCLWU:
2046 case RISCV::PseudoCCLD:
2047 case RISCV::PseudoCCQC_LI:
2048 return get(Opcode: MI.getOperand(i: MI.getNumExplicitOperands() - 3).getImm())
2049 .getSize() +
2050 4;
2051 case RISCV::PseudoCCQC_E_LI:
2052 case RISCV::PseudoCCQC_E_LB:
2053 case RISCV::PseudoCCQC_E_LH:
2054 case RISCV::PseudoCCQC_E_LW:
2055 case RISCV::PseudoCCQC_E_LHU:
2056 case RISCV::PseudoCCQC_E_LBU:
2057 return get(Opcode: MI.getOperand(i: MI.getNumExplicitOperands() - 3).getImm())
2058 .getSize() +
2059 6;
2060 case TargetOpcode::STACKMAP:
2061 // The upper bound for a stackmap intrinsic is the full length of its shadow
2062 return StackMapOpers(&MI).getNumPatchBytes();
2063 case TargetOpcode::PATCHPOINT:
2064 // The size of the patchpoint intrinsic is the number of bytes requested
2065 return PatchPointOpers(&MI).getNumPatchBytes();
2066 case TargetOpcode::STATEPOINT: {
2067 // The size of the statepoint intrinsic is the number of bytes requested
2068 unsigned NumBytes = StatepointOpers(&MI).getNumPatchBytes();
2069 // No patch bytes means at most a PseudoCall is emitted
2070 return std::max(a: NumBytes, b: 8U);
2071 }
2072 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
2073 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
2074 case TargetOpcode::PATCHABLE_TAIL_CALL: {
2075 const MachineFunction &MF = *MI.getParent()->getParent();
2076 const Function &F = MF.getFunction();
2077 if (Opcode == TargetOpcode::PATCHABLE_FUNCTION_ENTER &&
2078 F.hasFnAttribute(Kind: "patchable-function-entry")) {
2079 unsigned Num;
2080 if (F.getFnAttribute(Kind: "patchable-function-entry")
2081 .getValueAsString()
2082 .getAsInteger(Radix: 10, Result&: Num))
2083 return get(Opcode).getSize();
2084
2085 // Number of C.NOP or NOP
2086 return (STI.hasStdExtZca() ? 2 : 4) * Num;
2087 }
2088 // XRay uses C.JAL + 21 or 33 C.NOP for each sled in RV32 and RV64,
2089 // respectively.
2090 return STI.is64Bit() ? 68 : 44;
2091 }
2092 default:
2093 return get(Opcode).getSize();
2094 }
2095}
2096
2097unsigned RISCVInstrInfo::getInstBundleLength(const MachineInstr &MI) const {
2098 unsigned Size = 0;
2099 MachineBasicBlock::const_instr_iterator I = MI.getIterator();
2100 MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
2101 while (++I != E && I->isInsideBundle()) {
2102 assert(!I->isBundle() && "No nested bundle!");
2103 Size += getInstSizeInBytes(MI: *I);
2104 }
2105 return Size;
2106}
2107
2108bool RISCVInstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const {
2109 const unsigned Opcode = MI.getOpcode();
2110 switch (Opcode) {
2111 default:
2112 break;
2113 case RISCV::FSGNJ_D:
2114 case RISCV::FSGNJ_S:
2115 case RISCV::FSGNJ_H:
2116 case RISCV::FSGNJ_D_INX:
2117 case RISCV::FSGNJ_D_IN32X:
2118 case RISCV::FSGNJ_S_INX:
2119 case RISCV::FSGNJ_H_INX:
2120 // The canonical floating-point move is fsgnj rd, rs, rs.
2121 return MI.getOperand(i: 1).isReg() && MI.getOperand(i: 2).isReg() &&
2122 MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg();
2123 case RISCV::ADDI:
2124 case RISCV::ORI:
2125 case RISCV::XORI:
2126 return (MI.getOperand(i: 1).isReg() &&
2127 MI.getOperand(i: 1).getReg() == RISCV::X0) ||
2128 (MI.getOperand(i: 2).isImm() && MI.getOperand(i: 2).getImm() == 0);
2129 }
2130 return MI.isAsCheapAsAMove();
2131}
2132
2133std::optional<DestSourcePair>
2134RISCVInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
2135 if (MI.isMoveReg())
2136 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2137 switch (MI.getOpcode()) {
2138 default:
2139 break;
2140 case RISCV::ADD:
2141 case RISCV::OR:
2142 case RISCV::XOR:
2143 if (MI.getOperand(i: 1).isReg() && MI.getOperand(i: 1).getReg() == RISCV::X0 &&
2144 MI.getOperand(i: 2).isReg())
2145 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 2)};
2146 if (MI.getOperand(i: 2).isReg() && MI.getOperand(i: 2).getReg() == RISCV::X0 &&
2147 MI.getOperand(i: 1).isReg())
2148 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2149 break;
2150 case RISCV::ADDI:
2151 // Operand 1 can be a frameindex but callers expect registers
2152 if (MI.getOperand(i: 1).isReg() && MI.getOperand(i: 2).isImm() &&
2153 MI.getOperand(i: 2).getImm() == 0)
2154 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2155 break;
2156 case RISCV::SUB:
2157 if (MI.getOperand(i: 2).isReg() && MI.getOperand(i: 2).getReg() == RISCV::X0 &&
2158 MI.getOperand(i: 1).isReg())
2159 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2160 break;
2161 case RISCV::SH1ADD:
2162 case RISCV::SH1ADD_UW:
2163 case RISCV::SH2ADD:
2164 case RISCV::SH2ADD_UW:
2165 case RISCV::SH3ADD:
2166 case RISCV::SH3ADD_UW:
2167 if (MI.getOperand(i: 1).isReg() && MI.getOperand(i: 1).getReg() == RISCV::X0 &&
2168 MI.getOperand(i: 2).isReg())
2169 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 2)};
2170 break;
2171 case RISCV::FSGNJ_D:
2172 case RISCV::FSGNJ_S:
2173 case RISCV::FSGNJ_H:
2174 case RISCV::FSGNJ_D_INX:
2175 case RISCV::FSGNJ_D_IN32X:
2176 case RISCV::FSGNJ_S_INX:
2177 case RISCV::FSGNJ_H_INX:
2178 // The canonical floating-point move is fsgnj rd, rs, rs.
2179 if (MI.getOperand(i: 1).isReg() && MI.getOperand(i: 2).isReg() &&
2180 MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg())
2181 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2182 break;
2183 }
2184 return std::nullopt;
2185}
2186
2187MachineTraceStrategy RISCVInstrInfo::getMachineCombinerTraceStrategy() const {
2188 if (ForceMachineCombinerStrategy.getNumOccurrences() == 0) {
2189 // The option is unused. Choose Local strategy only for in-order cores. When
2190 // scheduling model is unspecified, use MinInstrCount strategy as more
2191 // generic one.
2192 const auto &SchedModel = STI.getSchedModel();
2193 return (!SchedModel.hasInstrSchedModel() || SchedModel.isOutOfOrder())
2194 ? MachineTraceStrategy::TS_MinInstrCount
2195 : MachineTraceStrategy::TS_Local;
2196 }
2197 // The strategy was forced by the option.
2198 return ForceMachineCombinerStrategy;
2199}
2200
2201void RISCVInstrInfo::finalizeInsInstrs(
2202 MachineInstr &Root, unsigned &Pattern,
2203 SmallVectorImpl<MachineInstr *> &InsInstrs) const {
2204 int16_t FrmOpIdx =
2205 RISCV::getNamedOperandIdx(Opcode: Root.getOpcode(), Name: RISCV::OpName::frm);
2206 if (FrmOpIdx < 0) {
2207 assert(all_of(InsInstrs,
2208 [](MachineInstr *MI) {
2209 return RISCV::getNamedOperandIdx(MI->getOpcode(),
2210 RISCV::OpName::frm) < 0;
2211 }) &&
2212 "New instructions require FRM whereas the old one does not have it");
2213 return;
2214 }
2215
2216 const MachineOperand &FRM = Root.getOperand(i: FrmOpIdx);
2217 MachineFunction &MF = *Root.getMF();
2218
2219 for (auto *NewMI : InsInstrs) {
2220 // We'd already added the FRM operand.
2221 if (static_cast<unsigned>(RISCV::getNamedOperandIdx(
2222 Opcode: NewMI->getOpcode(), Name: RISCV::OpName::frm)) != NewMI->getNumOperands())
2223 continue;
2224 MachineInstrBuilder MIB(MF, NewMI);
2225 MIB.add(MO: FRM);
2226 if (FRM.getImm() == RISCVFPRndMode::DYN)
2227 MIB.addUse(RegNo: RISCV::FRM, Flags: RegState::Implicit);
2228 }
2229}
2230
2231static bool isFADD(unsigned Opc) {
2232 switch (Opc) {
2233 default:
2234 return false;
2235 case RISCV::FADD_H:
2236 case RISCV::FADD_S:
2237 case RISCV::FADD_D:
2238 return true;
2239 }
2240}
2241
2242static bool isFSUB(unsigned Opc) {
2243 switch (Opc) {
2244 default:
2245 return false;
2246 case RISCV::FSUB_H:
2247 case RISCV::FSUB_S:
2248 case RISCV::FSUB_D:
2249 return true;
2250 }
2251}
2252
2253static bool isFMUL(unsigned Opc) {
2254 switch (Opc) {
2255 default:
2256 return false;
2257 case RISCV::FMUL_H:
2258 case RISCV::FMUL_S:
2259 case RISCV::FMUL_D:
2260 return true;
2261 }
2262}
2263
2264bool RISCVInstrInfo::isVectorAssociativeAndCommutative(const MachineInstr &Inst,
2265 bool Invert) const {
2266#define OPCODE_LMUL_CASE(OPC) \
2267 case RISCV::OPC##_M1: \
2268 case RISCV::OPC##_M2: \
2269 case RISCV::OPC##_M4: \
2270 case RISCV::OPC##_M8: \
2271 case RISCV::OPC##_MF2: \
2272 case RISCV::OPC##_MF4: \
2273 case RISCV::OPC##_MF8
2274
2275#define OPCODE_LMUL_MASK_CASE(OPC) \
2276 case RISCV::OPC##_M1_MASK: \
2277 case RISCV::OPC##_M2_MASK: \
2278 case RISCV::OPC##_M4_MASK: \
2279 case RISCV::OPC##_M8_MASK: \
2280 case RISCV::OPC##_MF2_MASK: \
2281 case RISCV::OPC##_MF4_MASK: \
2282 case RISCV::OPC##_MF8_MASK
2283
2284 unsigned Opcode = Inst.getOpcode();
2285 if (Invert) {
2286 if (auto InvOpcode = getInverseOpcode(Opcode))
2287 Opcode = *InvOpcode;
2288 else
2289 return false;
2290 }
2291
2292 // clang-format off
2293 switch (Opcode) {
2294 default:
2295 return false;
2296 OPCODE_LMUL_CASE(PseudoVADD_VV):
2297 OPCODE_LMUL_MASK_CASE(PseudoVADD_VV):
2298 OPCODE_LMUL_CASE(PseudoVMUL_VV):
2299 OPCODE_LMUL_MASK_CASE(PseudoVMUL_VV):
2300 return true;
2301 }
2302 // clang-format on
2303
2304#undef OPCODE_LMUL_MASK_CASE
2305#undef OPCODE_LMUL_CASE
2306}
2307
2308bool RISCVInstrInfo::areRVVInstsReassociable(const MachineInstr &Root,
2309 const MachineInstr &Prev) const {
2310 if (!areOpcodesEqualOrInverse(Opcode1: Root.getOpcode(), Opcode2: Prev.getOpcode()))
2311 return false;
2312
2313 assert(Root.getMF() == Prev.getMF());
2314 const MachineRegisterInfo *MRI = &Root.getMF()->getRegInfo();
2315 const TargetRegisterInfo *TRI = MRI->getTargetRegisterInfo();
2316
2317 // Make sure vtype operands are also the same.
2318 const MCInstrDesc &Desc = get(Opcode: Root.getOpcode());
2319 const uint64_t TSFlags = Desc.TSFlags;
2320
2321 auto checkImmOperand = [&](unsigned OpIdx) {
2322 return Root.getOperand(i: OpIdx).getImm() == Prev.getOperand(i: OpIdx).getImm();
2323 };
2324
2325 auto checkRegOperand = [&](unsigned OpIdx) {
2326 return Root.getOperand(i: OpIdx).getReg() == Prev.getOperand(i: OpIdx).getReg();
2327 };
2328
2329 // PassThru
2330 // TODO: Potentially we can loosen the condition to consider Root to be
2331 // associable with Prev if Root has NoReg as passthru. In which case we
2332 // also need to loosen the condition on vector policies between these.
2333 if (!checkRegOperand(1))
2334 return false;
2335
2336 // SEW
2337 if (RISCVII::hasSEWOp(TSFlags) &&
2338 !checkImmOperand(RISCVII::getSEWOpNum(Desc)))
2339 return false;
2340
2341 // Mask
2342 if (RISCVII::usesMaskPolicy(TSFlags)) {
2343 const MachineBasicBlock *MBB = Root.getParent();
2344 const MachineBasicBlock::const_reverse_iterator It1(&Root);
2345 const MachineBasicBlock::const_reverse_iterator It2(&Prev);
2346 Register MI1VReg;
2347
2348 bool SeenMI2 = false;
2349 for (auto End = MBB->rend(), It = It1; It != End; ++It) {
2350 if (It == It2) {
2351 SeenMI2 = true;
2352 if (!MI1VReg.isValid())
2353 // There is no V0 def between Root and Prev; they're sharing the
2354 // same V0.
2355 break;
2356 }
2357
2358 if (It->modifiesRegister(Reg: RISCV::V0, TRI)) {
2359 Register SrcReg = It->getOperand(i: 1).getReg();
2360 // If it's not VReg it'll be more difficult to track its defs, so
2361 // bailing out here just to be safe.
2362 if (!SrcReg.isVirtual())
2363 return false;
2364
2365 if (!MI1VReg.isValid()) {
2366 // This is the V0 def for Root.
2367 MI1VReg = SrcReg;
2368 continue;
2369 }
2370
2371 // Some random mask updates.
2372 if (!SeenMI2)
2373 continue;
2374
2375 // This is the V0 def for Prev; check if it's the same as that of
2376 // Root.
2377 if (MI1VReg != SrcReg)
2378 return false;
2379 else
2380 break;
2381 }
2382 }
2383
2384 // If we haven't encountered Prev, it's likely that this function was
2385 // called in a wrong way (e.g. Root is before Prev).
2386 assert(SeenMI2 && "Prev is expected to appear before Root");
2387 }
2388
2389 // Tail / Mask policies
2390 if (RISCVII::hasVecPolicyOp(TSFlags) &&
2391 !checkImmOperand(RISCVII::getVecPolicyOpNum(Desc)))
2392 return false;
2393
2394 // VL
2395 if (RISCVII::hasVLOp(TSFlags)) {
2396 unsigned OpIdx = RISCVII::getVLOpNum(Desc);
2397 const MachineOperand &Op1 = Root.getOperand(i: OpIdx);
2398 const MachineOperand &Op2 = Prev.getOperand(i: OpIdx);
2399 if (Op1.getType() != Op2.getType())
2400 return false;
2401 switch (Op1.getType()) {
2402 case MachineOperand::MO_Register:
2403 if (Op1.getReg() != Op2.getReg())
2404 return false;
2405 break;
2406 case MachineOperand::MO_Immediate:
2407 if (Op1.getImm() != Op2.getImm())
2408 return false;
2409 break;
2410 default:
2411 llvm_unreachable("Unrecognized VL operand type");
2412 }
2413 }
2414
2415 // Rounding modes
2416 if (int Idx = RISCVII::getFRMOpNum(Desc); Idx >= 0 && !checkImmOperand(Idx))
2417 return false;
2418 if (int Idx = RISCVII::getVXRMOpNum(Desc); Idx >= 0 && !checkImmOperand(Idx))
2419 return false;
2420
2421 return true;
2422}
2423
2424// Most of our RVV pseudos have passthru operand, so the real operands
2425// start from index = 2.
2426bool RISCVInstrInfo::hasReassociableVectorSibling(const MachineInstr &Inst,
2427 bool &Commuted) const {
2428 const MachineBasicBlock *MBB = Inst.getParent();
2429 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
2430 assert(RISCVII::isFirstDefTiedToFirstUse(get(Inst.getOpcode())) &&
2431 "Expect the present of passthrough operand.");
2432 MachineInstr *MI1 = MRI.getUniqueVRegDef(Reg: Inst.getOperand(i: 2).getReg());
2433 MachineInstr *MI2 = MRI.getUniqueVRegDef(Reg: Inst.getOperand(i: 3).getReg());
2434
2435 // If only one operand has the same or inverse opcode and it's the second
2436 // source operand, the operands must be commuted.
2437 Commuted = !areRVVInstsReassociable(Root: Inst, Prev: *MI1) &&
2438 areRVVInstsReassociable(Root: Inst, Prev: *MI2);
2439 if (Commuted)
2440 std::swap(a&: MI1, b&: MI2);
2441
2442 return areRVVInstsReassociable(Root: Inst, Prev: *MI1) &&
2443 (isVectorAssociativeAndCommutative(Inst: *MI1) ||
2444 isVectorAssociativeAndCommutative(Inst: *MI1, /* Invert */ true)) &&
2445 hasReassociableOperands(Inst: *MI1, MBB) &&
2446 MRI.hasOneNonDBGUse(RegNo: MI1->getOperand(i: 0).getReg());
2447}
2448
2449bool RISCVInstrInfo::hasReassociableOperands(
2450 const MachineInstr &Inst, const MachineBasicBlock *MBB) const {
2451 if (!isVectorAssociativeAndCommutative(Inst) &&
2452 !isVectorAssociativeAndCommutative(Inst, /*Invert=*/true))
2453 return TargetInstrInfo::hasReassociableOperands(Inst, MBB);
2454
2455 const MachineOperand &Op1 = Inst.getOperand(i: 2);
2456 const MachineOperand &Op2 = Inst.getOperand(i: 3);
2457 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
2458
2459 // We need virtual register definitions for the operands that we will
2460 // reassociate.
2461 MachineInstr *MI1 = nullptr;
2462 MachineInstr *MI2 = nullptr;
2463 if (Op1.isReg() && Op1.getReg().isVirtual())
2464 MI1 = MRI.getUniqueVRegDef(Reg: Op1.getReg());
2465 if (Op2.isReg() && Op2.getReg().isVirtual())
2466 MI2 = MRI.getUniqueVRegDef(Reg: Op2.getReg());
2467
2468 // And at least one operand must be defined in MBB.
2469 return MI1 && MI2 && (MI1->getParent() == MBB || MI2->getParent() == MBB);
2470}
2471
2472void RISCVInstrInfo::getReassociateOperandIndices(
2473 const MachineInstr &Root, unsigned Pattern,
2474 std::array<unsigned, 5> &OperandIndices) const {
2475 TargetInstrInfo::getReassociateOperandIndices(Root, Pattern, OperandIndices);
2476 if (RISCV::getRVVMCOpcode(RVVPseudoOpcode: Root.getOpcode())) {
2477 // Skip the passthrough operand, so increment all indices by one.
2478 for (unsigned I = 0; I < 5; ++I)
2479 ++OperandIndices[I];
2480 }
2481}
2482
2483bool RISCVInstrInfo::hasReassociableSibling(const MachineInstr &Inst,
2484 bool &Commuted) const {
2485 if (isVectorAssociativeAndCommutative(Inst) ||
2486 isVectorAssociativeAndCommutative(Inst, /*Invert=*/true))
2487 return hasReassociableVectorSibling(Inst, Commuted);
2488
2489 if (!TargetInstrInfo::hasReassociableSibling(Inst, Commuted))
2490 return false;
2491
2492 const MachineRegisterInfo &MRI = Inst.getMF()->getRegInfo();
2493 unsigned OperandIdx = Commuted ? 2 : 1;
2494 const MachineInstr &Sibling =
2495 *MRI.getVRegDef(Reg: Inst.getOperand(i: OperandIdx).getReg());
2496
2497 int16_t InstFrmOpIdx =
2498 RISCV::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name: RISCV::OpName::frm);
2499 int16_t SiblingFrmOpIdx =
2500 RISCV::getNamedOperandIdx(Opcode: Sibling.getOpcode(), Name: RISCV::OpName::frm);
2501
2502 return (InstFrmOpIdx < 0 && SiblingFrmOpIdx < 0) ||
2503 RISCV::hasEqualFRM(MI1: Inst, MI2: Sibling);
2504}
2505
2506bool RISCVInstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst,
2507 bool Invert) const {
2508 if (isVectorAssociativeAndCommutative(Inst, Invert))
2509 return true;
2510
2511 unsigned Opc = Inst.getOpcode();
2512 if (Invert) {
2513 auto InverseOpcode = getInverseOpcode(Opcode: Opc);
2514 if (!InverseOpcode)
2515 return false;
2516 Opc = *InverseOpcode;
2517 }
2518
2519 if (isFADD(Opc) || isFMUL(Opc))
2520 return Inst.getFlag(Flag: MachineInstr::MIFlag::FmReassoc) &&
2521 Inst.getFlag(Flag: MachineInstr::MIFlag::FmNsz);
2522
2523 switch (Opc) {
2524 default:
2525 return false;
2526 case RISCV::ADD:
2527 case RISCV::ADDW:
2528 case RISCV::AND:
2529 case RISCV::OR:
2530 case RISCV::XOR:
2531 // From RISC-V ISA spec, if both the high and low bits of the same product
2532 // are required, then the recommended code sequence is:
2533 //
2534 // MULH[[S]U] rdh, rs1, rs2
2535 // MUL rdl, rs1, rs2
2536 // (source register specifiers must be in same order and rdh cannot be the
2537 // same as rs1 or rs2)
2538 //
2539 // Microarchitectures can then fuse these into a single multiply operation
2540 // instead of performing two separate multiplies.
2541 // MachineCombiner may reassociate MUL operands and lose the fusion
2542 // opportunity.
2543 case RISCV::MUL:
2544 case RISCV::MULW:
2545 case RISCV::MIN:
2546 case RISCV::MINU:
2547 case RISCV::MAX:
2548 case RISCV::MAXU:
2549 case RISCV::FMIN_H:
2550 case RISCV::FMIN_S:
2551 case RISCV::FMIN_D:
2552 case RISCV::FMAX_H:
2553 case RISCV::FMAX_S:
2554 case RISCV::FMAX_D:
2555 return true;
2556 }
2557
2558 return false;
2559}
2560
2561std::optional<unsigned>
2562RISCVInstrInfo::getInverseOpcode(unsigned Opcode) const {
2563#define RVV_OPC_LMUL_CASE(OPC, INV) \
2564 case RISCV::OPC##_M1: \
2565 return RISCV::INV##_M1; \
2566 case RISCV::OPC##_M2: \
2567 return RISCV::INV##_M2; \
2568 case RISCV::OPC##_M4: \
2569 return RISCV::INV##_M4; \
2570 case RISCV::OPC##_M8: \
2571 return RISCV::INV##_M8; \
2572 case RISCV::OPC##_MF2: \
2573 return RISCV::INV##_MF2; \
2574 case RISCV::OPC##_MF4: \
2575 return RISCV::INV##_MF4; \
2576 case RISCV::OPC##_MF8: \
2577 return RISCV::INV##_MF8
2578
2579#define RVV_OPC_LMUL_MASK_CASE(OPC, INV) \
2580 case RISCV::OPC##_M1_MASK: \
2581 return RISCV::INV##_M1_MASK; \
2582 case RISCV::OPC##_M2_MASK: \
2583 return RISCV::INV##_M2_MASK; \
2584 case RISCV::OPC##_M4_MASK: \
2585 return RISCV::INV##_M4_MASK; \
2586 case RISCV::OPC##_M8_MASK: \
2587 return RISCV::INV##_M8_MASK; \
2588 case RISCV::OPC##_MF2_MASK: \
2589 return RISCV::INV##_MF2_MASK; \
2590 case RISCV::OPC##_MF4_MASK: \
2591 return RISCV::INV##_MF4_MASK; \
2592 case RISCV::OPC##_MF8_MASK: \
2593 return RISCV::INV##_MF8_MASK
2594
2595 switch (Opcode) {
2596 default:
2597 return std::nullopt;
2598 case RISCV::FADD_H:
2599 return RISCV::FSUB_H;
2600 case RISCV::FADD_S:
2601 return RISCV::FSUB_S;
2602 case RISCV::FADD_D:
2603 return RISCV::FSUB_D;
2604 case RISCV::FSUB_H:
2605 return RISCV::FADD_H;
2606 case RISCV::FSUB_S:
2607 return RISCV::FADD_S;
2608 case RISCV::FSUB_D:
2609 return RISCV::FADD_D;
2610 case RISCV::ADD:
2611 return RISCV::SUB;
2612 case RISCV::SUB:
2613 return RISCV::ADD;
2614 case RISCV::ADDW:
2615 return RISCV::SUBW;
2616 case RISCV::SUBW:
2617 return RISCV::ADDW;
2618 // clang-format off
2619 RVV_OPC_LMUL_CASE(PseudoVADD_VV, PseudoVSUB_VV);
2620 RVV_OPC_LMUL_MASK_CASE(PseudoVADD_VV, PseudoVSUB_VV);
2621 RVV_OPC_LMUL_CASE(PseudoVSUB_VV, PseudoVADD_VV);
2622 RVV_OPC_LMUL_MASK_CASE(PseudoVSUB_VV, PseudoVADD_VV);
2623 // clang-format on
2624 }
2625
2626#undef RVV_OPC_LMUL_MASK_CASE
2627#undef RVV_OPC_LMUL_CASE
2628}
2629
2630static bool canCombineFPFusedMultiply(const MachineInstr &Root,
2631 const MachineOperand &MO,
2632 bool DoRegPressureReduce) {
2633 if (!MO.isReg() || !MO.getReg().isVirtual())
2634 return false;
2635 const MachineRegisterInfo &MRI = Root.getMF()->getRegInfo();
2636 MachineInstr *MI = MRI.getVRegDef(Reg: MO.getReg());
2637 if (!MI || !isFMUL(Opc: MI->getOpcode()))
2638 return false;
2639
2640 if (!Root.getFlag(Flag: MachineInstr::MIFlag::FmContract) ||
2641 !MI->getFlag(Flag: MachineInstr::MIFlag::FmContract))
2642 return false;
2643
2644 // Try combining even if fmul has more than one use as it eliminates
2645 // dependency between fadd(fsub) and fmul. However, it can extend liveranges
2646 // for fmul operands, so reject the transformation in register pressure
2647 // reduction mode.
2648 if (DoRegPressureReduce && !MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()))
2649 return false;
2650
2651 // Do not combine instructions from different basic blocks.
2652 if (Root.getParent() != MI->getParent())
2653 return false;
2654 return RISCV::hasEqualFRM(MI1: Root, MI2: *MI);
2655}
2656
2657static bool getFPFusedMultiplyPatterns(MachineInstr &Root,
2658 SmallVectorImpl<unsigned> &Patterns,
2659 bool DoRegPressureReduce) {
2660 unsigned Opc = Root.getOpcode();
2661 bool IsFAdd = isFADD(Opc);
2662 if (!IsFAdd && !isFSUB(Opc))
2663 return false;
2664 bool Added = false;
2665 if (canCombineFPFusedMultiply(Root, MO: Root.getOperand(i: 1),
2666 DoRegPressureReduce)) {
2667 Patterns.push_back(Elt: IsFAdd ? RISCVMachineCombinerPattern::FMADD_AX
2668 : RISCVMachineCombinerPattern::FMSUB);
2669 Added = true;
2670 }
2671 if (canCombineFPFusedMultiply(Root, MO: Root.getOperand(i: 2),
2672 DoRegPressureReduce)) {
2673 Patterns.push_back(Elt: IsFAdd ? RISCVMachineCombinerPattern::FMADD_XA
2674 : RISCVMachineCombinerPattern::FNMSUB);
2675 Added = true;
2676 }
2677 return Added;
2678}
2679
2680static bool getFPPatterns(MachineInstr &Root,
2681 SmallVectorImpl<unsigned> &Patterns,
2682 bool DoRegPressureReduce) {
2683 return getFPFusedMultiplyPatterns(Root, Patterns, DoRegPressureReduce);
2684}
2685
2686/// Utility routine that checks if \param MO is defined by an
2687/// \param CombineOpc instruction in the basic block \param MBB
2688static const MachineInstr *canCombine(const MachineBasicBlock &MBB,
2689 const MachineOperand &MO,
2690 unsigned CombineOpc) {
2691 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2692 const MachineInstr *MI = nullptr;
2693
2694 if (MO.isReg() && MO.getReg().isVirtual())
2695 MI = MRI.getUniqueVRegDef(Reg: MO.getReg());
2696 // And it needs to be in the trace (otherwise, it won't have a depth).
2697 if (!MI || MI->getParent() != &MBB || MI->getOpcode() != CombineOpc)
2698 return nullptr;
2699 // Must only used by the user we combine with.
2700 if (!MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()))
2701 return nullptr;
2702
2703 return MI;
2704}
2705
2706/// Utility routine that checks if \param MO is defined by a SLLI in \param
2707/// MBB that can be combined by splitting across 2 SHXADD instructions. The
2708/// first SHXADD shift amount is given by \param OuterShiftAmt.
2709static bool canCombineShiftIntoShXAdd(const MachineBasicBlock &MBB,
2710 const MachineOperand &MO,
2711 unsigned OuterShiftAmt) {
2712 const MachineInstr *ShiftMI = canCombine(MBB, MO, CombineOpc: RISCV::SLLI);
2713 if (!ShiftMI)
2714 return false;
2715
2716 unsigned InnerShiftAmt = ShiftMI->getOperand(i: 2).getImm();
2717 if (InnerShiftAmt < OuterShiftAmt || (InnerShiftAmt - OuterShiftAmt) > 3)
2718 return false;
2719
2720 return true;
2721}
2722
2723// Returns the shift amount from a SHXADD instruction. Returns 0 if the
2724// instruction is not a SHXADD.
2725static unsigned getSHXADDShiftAmount(unsigned Opc) {
2726 switch (Opc) {
2727 default:
2728 return 0;
2729 case RISCV::SH1ADD:
2730 return 1;
2731 case RISCV::SH2ADD:
2732 return 2;
2733 case RISCV::SH3ADD:
2734 return 3;
2735 }
2736}
2737
2738// Returns the shift amount from a SHXADD.UW instruction. Returns 0 if the
2739// instruction is not a SHXADD.UW.
2740static unsigned getSHXADDUWShiftAmount(unsigned Opc) {
2741 switch (Opc) {
2742 default:
2743 return 0;
2744 case RISCV::SH1ADD_UW:
2745 return 1;
2746 case RISCV::SH2ADD_UW:
2747 return 2;
2748 case RISCV::SH3ADD_UW:
2749 return 3;
2750 }
2751}
2752
2753// Look for opportunities to combine (sh3add Z, (add X, (slli Y, 5))) into
2754// (sh3add (sh2add Y, Z), X).
2755static bool getSHXADDPatterns(const MachineInstr &Root,
2756 SmallVectorImpl<unsigned> &Patterns) {
2757 unsigned ShiftAmt = getSHXADDShiftAmount(Opc: Root.getOpcode());
2758 if (!ShiftAmt)
2759 return false;
2760
2761 const MachineBasicBlock &MBB = *Root.getParent();
2762
2763 const MachineInstr *AddMI = canCombine(MBB, MO: Root.getOperand(i: 2), CombineOpc: RISCV::ADD);
2764 if (!AddMI)
2765 return false;
2766
2767 bool Found = false;
2768 if (canCombineShiftIntoShXAdd(MBB, MO: AddMI->getOperand(i: 1), OuterShiftAmt: ShiftAmt)) {
2769 Patterns.push_back(Elt: RISCVMachineCombinerPattern::SHXADD_ADD_SLLI_OP1);
2770 Found = true;
2771 }
2772 if (canCombineShiftIntoShXAdd(MBB, MO: AddMI->getOperand(i: 2), OuterShiftAmt: ShiftAmt)) {
2773 Patterns.push_back(Elt: RISCVMachineCombinerPattern::SHXADD_ADD_SLLI_OP2);
2774 Found = true;
2775 }
2776
2777 return Found;
2778}
2779
2780CombinerObjective RISCVInstrInfo::getCombinerObjective(unsigned Pattern) const {
2781 switch (Pattern) {
2782 case RISCVMachineCombinerPattern::FMADD_AX:
2783 case RISCVMachineCombinerPattern::FMADD_XA:
2784 case RISCVMachineCombinerPattern::FMSUB:
2785 case RISCVMachineCombinerPattern::FNMSUB:
2786 return CombinerObjective::MustReduceDepth;
2787 default:
2788 return TargetInstrInfo::getCombinerObjective(Pattern);
2789 }
2790}
2791
2792bool RISCVInstrInfo::getMachineCombinerPatterns(
2793 MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns,
2794 bool DoRegPressureReduce) const {
2795
2796 if (getFPPatterns(Root, Patterns, DoRegPressureReduce))
2797 return true;
2798
2799 if (getSHXADDPatterns(Root, Patterns))
2800 return true;
2801
2802 return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
2803 DoRegPressureReduce);
2804}
2805
2806static unsigned getFPFusedMultiplyOpcode(unsigned RootOpc, unsigned Pattern) {
2807 switch (RootOpc) {
2808 default:
2809 llvm_unreachable("Unexpected opcode");
2810 case RISCV::FADD_H:
2811 return RISCV::FMADD_H;
2812 case RISCV::FADD_S:
2813 return RISCV::FMADD_S;
2814 case RISCV::FADD_D:
2815 return RISCV::FMADD_D;
2816 case RISCV::FSUB_H:
2817 return Pattern == RISCVMachineCombinerPattern::FMSUB ? RISCV::FMSUB_H
2818 : RISCV::FNMSUB_H;
2819 case RISCV::FSUB_S:
2820 return Pattern == RISCVMachineCombinerPattern::FMSUB ? RISCV::FMSUB_S
2821 : RISCV::FNMSUB_S;
2822 case RISCV::FSUB_D:
2823 return Pattern == RISCVMachineCombinerPattern::FMSUB ? RISCV::FMSUB_D
2824 : RISCV::FNMSUB_D;
2825 }
2826}
2827
2828static unsigned getAddendOperandIdx(unsigned Pattern) {
2829 switch (Pattern) {
2830 default:
2831 llvm_unreachable("Unexpected pattern");
2832 case RISCVMachineCombinerPattern::FMADD_AX:
2833 case RISCVMachineCombinerPattern::FMSUB:
2834 return 2;
2835 case RISCVMachineCombinerPattern::FMADD_XA:
2836 case RISCVMachineCombinerPattern::FNMSUB:
2837 return 1;
2838 }
2839}
2840
2841static void combineFPFusedMultiply(MachineInstr &Root, MachineInstr &Prev,
2842 unsigned Pattern,
2843 SmallVectorImpl<MachineInstr *> &InsInstrs,
2844 SmallVectorImpl<MachineInstr *> &DelInstrs) {
2845 MachineFunction *MF = Root.getMF();
2846 MachineRegisterInfo &MRI = MF->getRegInfo();
2847 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
2848
2849 MachineOperand &Mul1 = Prev.getOperand(i: 1);
2850 MachineOperand &Mul2 = Prev.getOperand(i: 2);
2851 MachineOperand &Dst = Root.getOperand(i: 0);
2852 MachineOperand &Addend = Root.getOperand(i: getAddendOperandIdx(Pattern));
2853
2854 Register DstReg = Dst.getReg();
2855 unsigned FusedOpc = getFPFusedMultiplyOpcode(RootOpc: Root.getOpcode(), Pattern);
2856 uint32_t IntersectedFlags = Root.getFlags() & Prev.getFlags();
2857 DebugLoc MergedLoc =
2858 DILocation::getMergedLocation(LocA: Root.getDebugLoc(), LocB: Prev.getDebugLoc());
2859
2860 bool Mul1IsKill = Mul1.isKill();
2861 bool Mul2IsKill = Mul2.isKill();
2862 bool AddendIsKill = Addend.isKill();
2863
2864 // We need to clear kill flags since we may be extending the live range past
2865 // a kill. If the mul had kill flags, we can preserve those since we know
2866 // where the previous range stopped.
2867 MRI.clearKillFlags(Reg: Mul1.getReg());
2868 MRI.clearKillFlags(Reg: Mul2.getReg());
2869
2870 MachineInstrBuilder MIB =
2871 BuildMI(MF&: *MF, MIMD: MergedLoc, MCID: TII->get(Opcode: FusedOpc), DestReg: DstReg)
2872 .addReg(RegNo: Mul1.getReg(), Flags: getKillRegState(B: Mul1IsKill))
2873 .addReg(RegNo: Mul2.getReg(), Flags: getKillRegState(B: Mul2IsKill))
2874 .addReg(RegNo: Addend.getReg(), Flags: getKillRegState(B: AddendIsKill))
2875 .setMIFlags(IntersectedFlags);
2876
2877 InsInstrs.push_back(Elt: MIB);
2878 if (MRI.hasOneNonDBGUse(RegNo: Prev.getOperand(i: 0).getReg()))
2879 DelInstrs.push_back(Elt: &Prev);
2880 DelInstrs.push_back(Elt: &Root);
2881}
2882
2883// Combine patterns like (sh3add Z, (add X, (slli Y, 5))) to
2884// (sh3add (sh2add Y, Z), X) if the shift amount can be split across two
2885// shXadd instructions. The outer shXadd keeps its original opcode.
2886static void
2887genShXAddAddShift(MachineInstr &Root, unsigned AddOpIdx,
2888 SmallVectorImpl<MachineInstr *> &InsInstrs,
2889 SmallVectorImpl<MachineInstr *> &DelInstrs,
2890 DenseMap<Register, unsigned> &InstrIdxForVirtReg) {
2891 MachineFunction *MF = Root.getMF();
2892 MachineRegisterInfo &MRI = MF->getRegInfo();
2893 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
2894
2895 unsigned OuterShiftAmt = getSHXADDShiftAmount(Opc: Root.getOpcode());
2896 assert(OuterShiftAmt != 0 && "Unexpected opcode");
2897
2898 MachineInstr *AddMI = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: 2).getReg());
2899 MachineInstr *ShiftMI =
2900 MRI.getUniqueVRegDef(Reg: AddMI->getOperand(i: AddOpIdx).getReg());
2901
2902 unsigned InnerShiftAmt = ShiftMI->getOperand(i: 2).getImm();
2903 assert(InnerShiftAmt >= OuterShiftAmt && "Unexpected shift amount");
2904
2905 unsigned InnerOpc;
2906 switch (InnerShiftAmt - OuterShiftAmt) {
2907 default:
2908 llvm_unreachable("Unexpected shift amount");
2909 case 0:
2910 InnerOpc = RISCV::ADD;
2911 break;
2912 case 1:
2913 InnerOpc = RISCV::SH1ADD;
2914 break;
2915 case 2:
2916 InnerOpc = RISCV::SH2ADD;
2917 break;
2918 case 3:
2919 InnerOpc = RISCV::SH3ADD;
2920 break;
2921 }
2922
2923 const MachineOperand &X = AddMI->getOperand(i: 3 - AddOpIdx);
2924 const MachineOperand &Y = ShiftMI->getOperand(i: 1);
2925 const MachineOperand &Z = Root.getOperand(i: 1);
2926
2927 Register NewVR = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2928
2929 auto MIB1 = BuildMI(MF&: *MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: InnerOpc), DestReg: NewVR)
2930 .addReg(RegNo: Y.getReg(), Flags: getKillRegState(B: Y.isKill()))
2931 .addReg(RegNo: Z.getReg(), Flags: getKillRegState(B: Z.isKill()));
2932 auto MIB2 = BuildMI(MF&: *MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: Root.getOpcode()),
2933 DestReg: Root.getOperand(i: 0).getReg())
2934 .addReg(RegNo: NewVR, Flags: RegState::Kill)
2935 .addReg(RegNo: X.getReg(), Flags: getKillRegState(B: X.isKill()));
2936
2937 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
2938 InsInstrs.push_back(Elt: MIB1);
2939 InsInstrs.push_back(Elt: MIB2);
2940 DelInstrs.push_back(Elt: ShiftMI);
2941 DelInstrs.push_back(Elt: AddMI);
2942 DelInstrs.push_back(Elt: &Root);
2943}
2944
2945void RISCVInstrInfo::genAlternativeCodeSequence(
2946 MachineInstr &Root, unsigned Pattern,
2947 SmallVectorImpl<MachineInstr *> &InsInstrs,
2948 SmallVectorImpl<MachineInstr *> &DelInstrs,
2949 DenseMap<Register, unsigned> &InstrIdxForVirtReg) const {
2950 MachineRegisterInfo &MRI = Root.getMF()->getRegInfo();
2951 switch (Pattern) {
2952 default:
2953 TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs,
2954 DelInstrs, InstIdxForVirtReg&: InstrIdxForVirtReg);
2955 return;
2956 case RISCVMachineCombinerPattern::FMADD_AX:
2957 case RISCVMachineCombinerPattern::FMSUB: {
2958 MachineInstr &Prev = *MRI.getVRegDef(Reg: Root.getOperand(i: 1).getReg());
2959 combineFPFusedMultiply(Root, Prev, Pattern, InsInstrs, DelInstrs);
2960 return;
2961 }
2962 case RISCVMachineCombinerPattern::FMADD_XA:
2963 case RISCVMachineCombinerPattern::FNMSUB: {
2964 MachineInstr &Prev = *MRI.getVRegDef(Reg: Root.getOperand(i: 2).getReg());
2965 combineFPFusedMultiply(Root, Prev, Pattern, InsInstrs, DelInstrs);
2966 return;
2967 }
2968 case RISCVMachineCombinerPattern::SHXADD_ADD_SLLI_OP1:
2969 genShXAddAddShift(Root, AddOpIdx: 1, InsInstrs, DelInstrs, InstrIdxForVirtReg);
2970 return;
2971 case RISCVMachineCombinerPattern::SHXADD_ADD_SLLI_OP2:
2972 genShXAddAddShift(Root, AddOpIdx: 2, InsInstrs, DelInstrs, InstrIdxForVirtReg);
2973 return;
2974 }
2975}
2976
2977bool RISCVInstrInfo::verifyInstruction(const MachineInstr &MI,
2978 StringRef &ErrInfo) const {
2979 MCInstrDesc const &Desc = MI.getDesc();
2980
2981 for (const auto &[Index, Operand] : enumerate(First: Desc.operands())) {
2982 const MachineOperand &MO = MI.getOperand(i: Index);
2983 unsigned OpType = Operand.OperandType;
2984 switch (OpType) {
2985 default:
2986 if (OpType >= RISCVOp::OPERAND_FIRST_RISCV_IMM &&
2987 OpType <= RISCVOp::OPERAND_LAST_RISCV_IMM) {
2988 if (!MO.isImm()) {
2989 ErrInfo = "Expected an immediate operand.";
2990 return false;
2991 }
2992 int64_t Imm = MO.getImm();
2993 bool Ok;
2994 switch (OpType) {
2995 default:
2996 llvm_unreachable("Unexpected operand type");
2997
2998#define CASE_OPERAND_UIMM(NUM) \
2999 case RISCVOp::OPERAND_UIMM##NUM: \
3000 Ok = isUInt<NUM>(Imm); \
3001 break;
3002#define CASE_OPERAND_UIMM_LSB_ZEROS(BITS, SUFFIX) \
3003 case RISCVOp::OPERAND_UIMM##BITS##_LSB##SUFFIX: { \
3004 constexpr size_t NumZeros = sizeof(#SUFFIX) - 1; \
3005 Ok = isShiftedUInt<BITS - NumZeros, NumZeros>(Imm); \
3006 break; \
3007 }
3008#define CASE_OPERAND_SIMM(NUM) \
3009 case RISCVOp::OPERAND_SIMM##NUM: \
3010 Ok = isInt<NUM>(Imm); \
3011 break;
3012 // clang-format off
3013 CASE_OPERAND_UIMM(1)
3014 CASE_OPERAND_UIMM(2)
3015 CASE_OPERAND_UIMM(3)
3016 CASE_OPERAND_UIMM(4)
3017 CASE_OPERAND_UIMM(5)
3018 CASE_OPERAND_UIMM(6)
3019 CASE_OPERAND_UIMM(7)
3020 CASE_OPERAND_UIMM(8)
3021 CASE_OPERAND_UIMM(9)
3022 CASE_OPERAND_UIMM(10)
3023 CASE_OPERAND_UIMM(12)
3024 CASE_OPERAND_UIMM(16)
3025 CASE_OPERAND_UIMM(32)
3026 CASE_OPERAND_UIMM(48)
3027 CASE_OPERAND_UIMM(64)
3028 CASE_OPERAND_UIMM_LSB_ZEROS(2, 0)
3029 CASE_OPERAND_UIMM_LSB_ZEROS(5, 0)
3030 CASE_OPERAND_UIMM_LSB_ZEROS(6, 0)
3031 CASE_OPERAND_UIMM_LSB_ZEROS(7, 00)
3032 CASE_OPERAND_UIMM_LSB_ZEROS(7, 000)
3033 CASE_OPERAND_UIMM_LSB_ZEROS(8, 00)
3034 CASE_OPERAND_UIMM_LSB_ZEROS(8, 000)
3035 CASE_OPERAND_UIMM_LSB_ZEROS(9, 000)
3036 // clang-format on
3037 case RISCVOp::OPERAND_UIMM5_NONZERO:
3038 Ok = isUInt<5>(x: Imm) && (Imm != 0);
3039 break;
3040 case RISCVOp::OPERAND_UIMM5_GT3:
3041 Ok = isUInt<5>(x: Imm) && (Imm > 3);
3042 break;
3043 case RISCVOp::OPERAND_UIMM5_PLUS1:
3044 Ok = Imm >= 1 && Imm <= 32;
3045 break;
3046 case RISCVOp::OPERAND_UIMM6_PLUS1:
3047 Ok = Imm >= 1 && Imm <= 64;
3048 break;
3049 case RISCVOp::OPERAND_UIMM8_GE32:
3050 Ok = isUInt<8>(x: Imm) && Imm >= 32;
3051 break;
3052 case RISCVOp::OPERAND_SIMM10_LSB0000_NONZERO:
3053 Ok = isShiftedInt<6, 4>(x: Imm) && (Imm != 0);
3054 break;
3055 case RISCVOp::OPERAND_UIMM10_LSB00_NONZERO:
3056 Ok = isShiftedUInt<8, 2>(x: Imm) && (Imm != 0);
3057 break;
3058 case RISCVOp::OPERAND_UIMM16_NONZERO:
3059 Ok = isUInt<16>(x: Imm) && (Imm != 0);
3060 break;
3061 case RISCVOp::OPERAND_THREE:
3062 Ok = Imm == 3;
3063 break;
3064 case RISCVOp::OPERAND_FOUR:
3065 Ok = Imm == 4;
3066 break;
3067 case RISCVOp::OPERAND_IMM5_ZIBI:
3068 Ok = (isUInt<5>(x: Imm) && Imm != 0) || Imm == -1;
3069 break;
3070 // clang-format off
3071 CASE_OPERAND_SIMM(5)
3072 CASE_OPERAND_SIMM(6)
3073 CASE_OPERAND_SIMM(8)
3074 CASE_OPERAND_SIMM(10)
3075 CASE_OPERAND_SIMM(11)
3076 CASE_OPERAND_SIMM(26)
3077 // clang-format on
3078 case RISCVOp::OPERAND_SIMM5_PLUS1:
3079 Ok = Imm >= -15 && Imm <= 16;
3080 break;
3081 case RISCVOp::OPERAND_SIMM5_NONZERO:
3082 Ok = isInt<5>(x: Imm) && (Imm != 0);
3083 break;
3084 case RISCVOp::OPERAND_SIMM6_NONZERO:
3085 Ok = Imm != 0 && isInt<6>(x: Imm);
3086 break;
3087 case RISCVOp::OPERAND_VTYPEI10:
3088 Ok = isUInt<10>(x: Imm);
3089 break;
3090 case RISCVOp::OPERAND_VTYPEI11:
3091 Ok = isUInt<11>(x: Imm);
3092 break;
3093 case RISCVOp::OPERAND_SIMM12_LSB00000:
3094 Ok = isShiftedInt<7, 5>(x: Imm);
3095 break;
3096 case RISCVOp::OPERAND_SIMM16_NONZERO:
3097 Ok = isInt<16>(x: Imm) && (Imm != 0);
3098 break;
3099 case RISCVOp::OPERAND_SIMM20_LI:
3100 Ok = isInt<20>(x: Imm);
3101 break;
3102 case RISCVOp::OPERAND_UIMMLOG2XLEN:
3103 Ok = STI.is64Bit() ? isUInt<6>(x: Imm) : isUInt<5>(x: Imm);
3104 break;
3105 case RISCVOp::OPERAND_UIMMLOG2XLEN_NONZERO:
3106 Ok = STI.is64Bit() ? isUInt<6>(x: Imm) : isUInt<5>(x: Imm);
3107 Ok = Ok && Imm != 0;
3108 break;
3109 case RISCVOp::OPERAND_CLUI_IMM:
3110 Ok = (isUInt<5>(x: Imm) && Imm != 0) || (Imm >= 0xfffe0 && Imm <= 0xfffff);
3111 break;
3112 case RISCVOp::OPERAND_RVKRNUM:
3113 Ok = Imm >= 0 && Imm <= 10;
3114 break;
3115 case RISCVOp::OPERAND_RVKRNUM_0_7:
3116 Ok = Imm >= 0 && Imm <= 7;
3117 break;
3118 case RISCVOp::OPERAND_RVKRNUM_1_10:
3119 Ok = Imm >= 1 && Imm <= 10;
3120 break;
3121 case RISCVOp::OPERAND_RVKRNUM_2_14:
3122 Ok = Imm >= 2 && Imm <= 14;
3123 break;
3124 case RISCVOp::OPERAND_RLIST:
3125 Ok = Imm >= RISCVZC::RA && Imm <= RISCVZC::RA_S0_S11;
3126 break;
3127 case RISCVOp::OPERAND_RLIST_S0:
3128 Ok = Imm >= RISCVZC::RA_S0 && Imm <= RISCVZC::RA_S0_S11;
3129 break;
3130 case RISCVOp::OPERAND_STACKADJ:
3131 Ok = Imm >= 0 && Imm <= 48 && Imm % 16 == 0;
3132 break;
3133 case RISCVOp::OPERAND_FRMARG:
3134 Ok = RISCVFPRndMode::isValidRoundingMode(Mode: Imm);
3135 break;
3136 case RISCVOp::OPERAND_RTZARG:
3137 Ok = Imm == RISCVFPRndMode::RTZ;
3138 break;
3139 case RISCVOp::OPERAND_COND_CODE:
3140 Ok = Imm >= 0 && Imm < RISCVCC::COND_INVALID;
3141 break;
3142 case RISCVOp::OPERAND_ATOMIC_ORDERING:
3143 Ok = isValidAtomicOrdering(I: Imm);
3144 break;
3145 case RISCVOp::OPERAND_VEC_POLICY:
3146 Ok = (Imm & (RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC)) ==
3147 Imm;
3148 break;
3149 case RISCVOp::OPERAND_SEW:
3150 Ok = (isUInt<5>(x: Imm) && RISCVVType::isValidSEW(SEW: 1 << Imm));
3151 break;
3152 case RISCVOp::OPERAND_SEW_MASK:
3153 Ok = Imm == 0;
3154 break;
3155 case RISCVOp::OPERAND_VEC_RM:
3156 assert(RISCVII::hasRoundModeOp(Desc.TSFlags));
3157 if (RISCVII::usesVXRM(TSFlags: Desc.TSFlags))
3158 Ok = isUInt<2>(x: Imm);
3159 else
3160 Ok = RISCVFPRndMode::isValidRoundingMode(Mode: Imm);
3161 break;
3162 case RISCVOp::OPERAND_XSFMM_VTYPE:
3163 Ok = RISCVVType::isValidXSfmmVType(VTypeI: Imm);
3164 break;
3165 case RISCVOp::OPERAND_XSFMM_TWIDEN:
3166 Ok = Imm == 1 || Imm == 2 || Imm == 4;
3167 break;
3168 }
3169 if (!Ok) {
3170 ErrInfo = "Invalid immediate";
3171 return false;
3172 }
3173 }
3174 break;
3175 case RISCVOp::OPERAND_SIMM12_LO:
3176 // TODO: We could be stricter about what non-register operands are
3177 // allowed.
3178 if (MO.isReg()) {
3179 ErrInfo = "Expected a non-register operand.";
3180 return false;
3181 }
3182 if (MO.isImm() && !isInt<12>(x: MO.getImm())) {
3183 ErrInfo = "Invalid immediate";
3184 return false;
3185 }
3186 break;
3187 case RISCVOp::OPERAND_UIMM20_LUI:
3188 case RISCVOp::OPERAND_UIMM20_AUIPC:
3189 // TODO: We could be stricter about what non-register operands are
3190 // allowed.
3191 if (MO.isReg()) {
3192 ErrInfo = "Expected a non-register operand.";
3193 return false;
3194 }
3195 if (MO.isImm() && !isUInt<20>(x: MO.getImm())) {
3196 ErrInfo = "Invalid immediate";
3197 return false;
3198 }
3199 break;
3200 case RISCVOp::OPERAND_BARE_SIMM32:
3201 // TODO: We could be stricter about what non-register operands are
3202 // allowed.
3203 if (MO.isReg()) {
3204 ErrInfo = "Expected a non-register operand.";
3205 return false;
3206 }
3207 if (MO.isImm() && !isInt<32>(x: MO.getImm())) {
3208 ErrInfo = "Invalid immediate";
3209 return false;
3210 }
3211 break;
3212 case RISCVOp::OPERAND_AVL:
3213 if (MO.isImm()) {
3214 int64_t Imm = MO.getImm();
3215 // VLMAX is represented as -1.
3216 if (!isUInt<5>(x: Imm) && Imm != -1) {
3217 ErrInfo = "Invalid immediate";
3218 return false;
3219 }
3220 } else if (!MO.isReg()) {
3221 ErrInfo = "Expected a register or immediate operand.";
3222 return false;
3223 }
3224 break;
3225 case RISCVOp::OPERAND_SFB_RHS:
3226 if (!MO.isReg() && !MO.isImm()) {
3227 ErrInfo = "Expected a register or immediate operand.";
3228 return false;
3229 }
3230 break;
3231 }
3232 }
3233
3234 const uint64_t TSFlags = Desc.TSFlags;
3235 if (RISCVII::hasVLOp(TSFlags)) {
3236 const MachineOperand &Op = MI.getOperand(i: RISCVII::getVLOpNum(Desc));
3237 if (!Op.isImm() && !Op.isReg()) {
3238 ErrInfo = "Invalid operand type for VL operand";
3239 return false;
3240 }
3241 if (Op.isReg() && Op.getReg().isValid()) {
3242 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
3243 auto *RC = MRI.getRegClass(Reg: Op.getReg());
3244 if (!RISCV::GPRNoX0RegClass.hasSubClassEq(RC)) {
3245 ErrInfo = "Invalid register class for VL operand";
3246 return false;
3247 }
3248 }
3249 if (!RISCVII::hasSEWOp(TSFlags)) {
3250 ErrInfo = "VL operand w/o SEW operand?";
3251 return false;
3252 }
3253 }
3254 if (RISCVII::hasSEWOp(TSFlags)) {
3255 unsigned OpIdx = RISCVII::getSEWOpNum(Desc);
3256 if (!MI.getOperand(i: OpIdx).isImm()) {
3257 ErrInfo = "SEW value expected to be an immediate";
3258 return false;
3259 }
3260 uint64_t Log2SEW = MI.getOperand(i: OpIdx).getImm();
3261 if (Log2SEW > 31) {
3262 ErrInfo = "Unexpected SEW value";
3263 return false;
3264 }
3265 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
3266 if (!RISCVVType::isValidSEW(SEW)) {
3267 ErrInfo = "Unexpected SEW value";
3268 return false;
3269 }
3270 }
3271 if (RISCVII::hasVecPolicyOp(TSFlags)) {
3272 unsigned OpIdx = RISCVII::getVecPolicyOpNum(Desc);
3273 if (!MI.getOperand(i: OpIdx).isImm()) {
3274 ErrInfo = "Policy operand expected to be an immediate";
3275 return false;
3276 }
3277 uint64_t Policy = MI.getOperand(i: OpIdx).getImm();
3278 if (Policy > (RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC)) {
3279 ErrInfo = "Invalid Policy Value";
3280 return false;
3281 }
3282 if (!RISCVII::hasVLOp(TSFlags)) {
3283 ErrInfo = "policy operand w/o VL operand?";
3284 return false;
3285 }
3286
3287 // VecPolicy operands can only exist on instructions with passthru/merge
3288 // arguments. Note that not all arguments with passthru have vec policy
3289 // operands- some instructions have implicit policies.
3290 unsigned UseOpIdx;
3291 if (!MI.isRegTiedToUseOperand(DefOpIdx: 0, UseOpIdx: &UseOpIdx)) {
3292 ErrInfo = "policy operand w/o tied operand?";
3293 return false;
3294 }
3295 }
3296
3297 if (int Idx = RISCVII::getFRMOpNum(Desc);
3298 Idx >= 0 && MI.getOperand(i: Idx).getImm() == RISCVFPRndMode::DYN &&
3299 !MI.readsRegister(Reg: RISCV::FRM, /*TRI=*/nullptr)) {
3300 ErrInfo = "dynamic rounding mode should read FRM";
3301 return false;
3302 }
3303
3304 return true;
3305}
3306
3307bool RISCVInstrInfo::canFoldIntoAddrMode(const MachineInstr &MemI, Register Reg,
3308 const MachineInstr &AddrI,
3309 ExtAddrMode &AM) const {
3310 switch (MemI.getOpcode()) {
3311 default:
3312 return false;
3313 case RISCV::LB:
3314 case RISCV::LBU:
3315 case RISCV::LH:
3316 case RISCV::LH_INX:
3317 case RISCV::LHU:
3318 case RISCV::LW:
3319 case RISCV::LW_INX:
3320 case RISCV::LWU:
3321 case RISCV::LD:
3322 case RISCV::LD_RV32:
3323 case RISCV::FLH:
3324 case RISCV::FLW:
3325 case RISCV::FLD:
3326 case RISCV::SB:
3327 case RISCV::SH:
3328 case RISCV::SH_INX:
3329 case RISCV::SW:
3330 case RISCV::SW_INX:
3331 case RISCV::SD:
3332 case RISCV::SD_RV32:
3333 case RISCV::FSH:
3334 case RISCV::FSW:
3335 case RISCV::FSD:
3336 break;
3337 }
3338
3339 if (MemI.getOperand(i: 0).getReg() == Reg)
3340 return false;
3341
3342 if (AddrI.getOpcode() != RISCV::ADDI || !AddrI.getOperand(i: 1).isReg() ||
3343 !AddrI.getOperand(i: 2).isImm())
3344 return false;
3345
3346 int64_t OldOffset = MemI.getOperand(i: 2).getImm();
3347 int64_t Disp = AddrI.getOperand(i: 2).getImm();
3348 int64_t NewOffset = OldOffset + Disp;
3349 if (!STI.is64Bit())
3350 NewOffset = SignExtend64<32>(x: NewOffset);
3351
3352 if (!isInt<12>(x: NewOffset))
3353 return false;
3354
3355 AM.BaseReg = AddrI.getOperand(i: 1).getReg();
3356 AM.ScaledReg = 0;
3357 AM.Scale = 0;
3358 AM.Displacement = NewOffset;
3359 AM.Form = ExtAddrMode::Formula::Basic;
3360 return true;
3361}
3362
3363MachineInstr *RISCVInstrInfo::emitLdStWithAddr(MachineInstr &MemI,
3364 const ExtAddrMode &AM) const {
3365
3366 const DebugLoc &DL = MemI.getDebugLoc();
3367 MachineBasicBlock &MBB = *MemI.getParent();
3368
3369 assert(AM.ScaledReg == 0 && AM.Scale == 0 &&
3370 "Addressing mode not supported for folding");
3371
3372 return BuildMI(BB&: MBB, I&: MemI, MIMD: DL, MCID: get(Opcode: MemI.getOpcode()))
3373 .addReg(RegNo: MemI.getOperand(i: 0).getReg(), Flags: getDefRegState(B: MemI.mayLoad()))
3374 .addReg(RegNo: AM.BaseReg)
3375 .addImm(Val: AM.Displacement)
3376 .setMemRefs(MemI.memoperands())
3377 .setMIFlags(MemI.getFlags());
3378}
3379
3380// TODO: At the moment, MIPS introduced paring of instructions operating with
3381// word or double word. This should be extended with more instructions when more
3382// vendors support load/store pairing.
3383bool RISCVInstrInfo::isPairableLdStInstOpc(unsigned Opc) {
3384 switch (Opc) {
3385 default:
3386 return false;
3387 case RISCV::SW:
3388 case RISCV::SD:
3389 case RISCV::LD:
3390 case RISCV::LW:
3391 return true;
3392 }
3393}
3394
3395bool RISCVInstrInfo::isLdStSafeToPair(const MachineInstr &LdSt,
3396 const TargetRegisterInfo *TRI) {
3397 // If this is a volatile load/store, don't mess with it.
3398 if (LdSt.hasOrderedMemoryRef() || LdSt.getNumExplicitOperands() != 3)
3399 return false;
3400
3401 if (LdSt.getOperand(i: 1).isFI())
3402 return true;
3403
3404 assert(LdSt.getOperand(1).isReg() && "Expected a reg operand.");
3405 // Can't cluster if the instruction modifies the base register
3406 // or it is update form. e.g. ld x5,8(x5)
3407 if (LdSt.modifiesRegister(Reg: LdSt.getOperand(i: 1).getReg(), TRI))
3408 return false;
3409
3410 if (!LdSt.getOperand(i: 2).isImm())
3411 return false;
3412
3413 return true;
3414}
3415
3416bool RISCVInstrInfo::getMemOperandsWithOffsetWidth(
3417 const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
3418 int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
3419 const TargetRegisterInfo *TRI) const {
3420 if (!LdSt.mayLoadOrStore())
3421 return false;
3422
3423 // Conservatively, only handle scalar loads/stores for now.
3424 switch (LdSt.getOpcode()) {
3425 case RISCV::LB:
3426 case RISCV::LBU:
3427 case RISCV::SB:
3428 case RISCV::LH:
3429 case RISCV::LH_INX:
3430 case RISCV::LHU:
3431 case RISCV::FLH:
3432 case RISCV::SH:
3433 case RISCV::SH_INX:
3434 case RISCV::FSH:
3435 case RISCV::LW:
3436 case RISCV::LW_INX:
3437 case RISCV::LWU:
3438 case RISCV::FLW:
3439 case RISCV::SW:
3440 case RISCV::SW_INX:
3441 case RISCV::FSW:
3442 case RISCV::LD:
3443 case RISCV::LD_RV32:
3444 case RISCV::FLD:
3445 case RISCV::SD:
3446 case RISCV::SD_RV32:
3447 case RISCV::FSD:
3448 break;
3449 default:
3450 return false;
3451 }
3452 const MachineOperand *BaseOp;
3453 OffsetIsScalable = false;
3454 if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width, TRI))
3455 return false;
3456 BaseOps.push_back(Elt: BaseOp);
3457 return true;
3458}
3459
3460// TODO: This was copied from SIInstrInfo. Could it be lifted to a common
3461// helper?
3462static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
3463 ArrayRef<const MachineOperand *> BaseOps1,
3464 const MachineInstr &MI2,
3465 ArrayRef<const MachineOperand *> BaseOps2) {
3466 // Only examine the first "base" operand of each instruction, on the
3467 // assumption that it represents the real base address of the memory access.
3468 // Other operands are typically offsets or indices from this base address.
3469 if (BaseOps1.front()->isIdenticalTo(Other: *BaseOps2.front()))
3470 return true;
3471
3472 if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
3473 return false;
3474
3475 auto MO1 = *MI1.memoperands_begin();
3476 auto MO2 = *MI2.memoperands_begin();
3477 if (MO1->getAddrSpace() != MO2->getAddrSpace())
3478 return false;
3479
3480 auto Base1 = MO1->getValue();
3481 auto Base2 = MO2->getValue();
3482 if (!Base1 || !Base2)
3483 return false;
3484 Base1 = getUnderlyingObject(V: Base1);
3485 Base2 = getUnderlyingObject(V: Base2);
3486
3487 if (isa<UndefValue>(Val: Base1) || isa<UndefValue>(Val: Base2))
3488 return false;
3489
3490 return Base1 == Base2;
3491}
3492
3493bool RISCVInstrInfo::shouldClusterMemOps(
3494 ArrayRef<const MachineOperand *> BaseOps1, int64_t Offset1,
3495 bool OffsetIsScalable1, ArrayRef<const MachineOperand *> BaseOps2,
3496 int64_t Offset2, bool OffsetIsScalable2, unsigned ClusterSize,
3497 unsigned NumBytes) const {
3498 // If the mem ops (to be clustered) do not have the same base ptr, then they
3499 // should not be clustered
3500 if (!BaseOps1.empty() && !BaseOps2.empty()) {
3501 const MachineInstr &FirstLdSt = *BaseOps1.front()->getParent();
3502 const MachineInstr &SecondLdSt = *BaseOps2.front()->getParent();
3503 if (!memOpsHaveSameBasePtr(MI1: FirstLdSt, BaseOps1, MI2: SecondLdSt, BaseOps2))
3504 return false;
3505 } else if (!BaseOps1.empty() || !BaseOps2.empty()) {
3506 // If only one base op is empty, they do not have the same base ptr
3507 return false;
3508 }
3509
3510 unsigned CacheLineSize =
3511 BaseOps1.front()->getParent()->getMF()->getSubtarget().getCacheLineSize();
3512 // Assume a cache line size of 64 bytes if no size is set in RISCVSubtarget.
3513 CacheLineSize = CacheLineSize ? CacheLineSize : 64;
3514 // Cluster if the memory operations are on the same or a neighbouring cache
3515 // line, but limit the maximum ClusterSize to avoid creating too much
3516 // additional register pressure.
3517 return ClusterSize <= 4 && std::abs(i: Offset1 - Offset2) < CacheLineSize;
3518}
3519
3520// Set BaseReg (the base register operand), Offset (the byte offset being
3521// accessed) and the access Width of the passed instruction that reads/writes
3522// memory. Returns false if the instruction does not read/write memory or the
3523// BaseReg/Offset/Width can't be determined. Is not guaranteed to always
3524// recognise base operands and offsets in all cases.
3525// TODO: Add an IsScalable bool ref argument (like the equivalent AArch64
3526// function) and set it as appropriate.
3527bool RISCVInstrInfo::getMemOperandWithOffsetWidth(
3528 const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset,
3529 LocationSize &Width, const TargetRegisterInfo *TRI) const {
3530 if (!LdSt.mayLoadOrStore())
3531 return false;
3532
3533 // Here we assume the standard RISC-V ISA, which uses a base+offset
3534 // addressing mode. You'll need to relax these conditions to support custom
3535 // load/store instructions.
3536 if (LdSt.getNumExplicitOperands() != 3)
3537 return false;
3538 if ((!LdSt.getOperand(i: 1).isReg() && !LdSt.getOperand(i: 1).isFI()) ||
3539 !LdSt.getOperand(i: 2).isImm())
3540 return false;
3541
3542 if (!LdSt.hasOneMemOperand())
3543 return false;
3544
3545 Width = (*LdSt.memoperands_begin())->getSize();
3546 BaseReg = &LdSt.getOperand(i: 1);
3547 Offset = LdSt.getOperand(i: 2).getImm();
3548 return true;
3549}
3550
3551bool RISCVInstrInfo::areMemAccessesTriviallyDisjoint(
3552 const MachineInstr &MIa, const MachineInstr &MIb) const {
3553 assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
3554 assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
3555
3556 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() ||
3557 MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
3558 return false;
3559
3560 // Retrieve the base register, offset from the base register and width. Width
3561 // is the size of memory that is being loaded/stored (e.g. 1, 2, 4). If
3562 // base registers are identical, and the offset of a lower memory access +
3563 // the width doesn't overlap the offset of a higher memory access,
3564 // then the memory accesses are different.
3565 const TargetRegisterInfo *TRI = STI.getRegisterInfo();
3566 const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
3567 int64_t OffsetA = 0, OffsetB = 0;
3568 LocationSize WidthA = LocationSize::precise(Value: 0),
3569 WidthB = LocationSize::precise(Value: 0);
3570 if (getMemOperandWithOffsetWidth(LdSt: MIa, BaseReg&: BaseOpA, Offset&: OffsetA, Width&: WidthA, TRI) &&
3571 getMemOperandWithOffsetWidth(LdSt: MIb, BaseReg&: BaseOpB, Offset&: OffsetB, Width&: WidthB, TRI)) {
3572 if (BaseOpA->isIdenticalTo(Other: *BaseOpB)) {
3573 int LowOffset = std::min(a: OffsetA, b: OffsetB);
3574 int HighOffset = std::max(a: OffsetA, b: OffsetB);
3575 LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
3576 if (LowWidth.hasValue() &&
3577 LowOffset + (int)LowWidth.getValue() <= HighOffset)
3578 return true;
3579 }
3580 }
3581 return false;
3582}
3583
3584std::pair<unsigned, unsigned>
3585RISCVInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
3586 const unsigned Mask = RISCVII::MO_DIRECT_FLAG_MASK;
3587 return std::make_pair(x: TF & Mask, y: TF & ~Mask);
3588}
3589
3590ArrayRef<std::pair<unsigned, const char *>>
3591RISCVInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
3592 using namespace RISCVII;
3593 static const std::pair<unsigned, const char *> TargetFlags[] = {
3594 {MO_CALL, "riscv-call"},
3595 {MO_LO, "riscv-lo"},
3596 {MO_HI, "riscv-hi"},
3597 {MO_PCREL_LO, "riscv-pcrel-lo"},
3598 {MO_PCREL_HI, "riscv-pcrel-hi"},
3599 {MO_GOT_HI, "riscv-got-hi"},
3600 {MO_TPREL_LO, "riscv-tprel-lo"},
3601 {MO_TPREL_HI, "riscv-tprel-hi"},
3602 {MO_TPREL_ADD, "riscv-tprel-add"},
3603 {MO_TLS_GOT_HI, "riscv-tls-got-hi"},
3604 {MO_TLS_GD_HI, "riscv-tls-gd-hi"},
3605 {MO_TLSDESC_HI, "riscv-tlsdesc-hi"},
3606 {MO_TLSDESC_LOAD_LO, "riscv-tlsdesc-load-lo"},
3607 {MO_TLSDESC_ADD_LO, "riscv-tlsdesc-add-lo"},
3608 {MO_TLSDESC_CALL, "riscv-tlsdesc-call"}};
3609 return ArrayRef(TargetFlags);
3610}
3611bool RISCVInstrInfo::isFunctionSafeToOutlineFrom(
3612 MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
3613 const Function &F = MF.getFunction();
3614
3615 // Can F be deduplicated by the linker? If it can, don't outline from it.
3616 if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage())
3617 return false;
3618
3619 // Don't outline from functions with section markings; the program could
3620 // expect that all the code is in the named section.
3621 if (F.hasSection())
3622 return false;
3623
3624 // It's safe to outline from MF.
3625 return true;
3626}
3627
3628bool RISCVInstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB,
3629 unsigned &Flags) const {
3630 // More accurate safety checking is done in getOutliningCandidateInfo.
3631 return TargetInstrInfo::isMBBSafeToOutlineFrom(MBB, Flags);
3632}
3633
3634// Enum values indicating how an outlined call should be constructed.
3635enum MachineOutlinerConstructionID {
3636 MachineOutlinerTailCall,
3637 MachineOutlinerDefault
3638};
3639
3640bool RISCVInstrInfo::shouldOutlineFromFunctionByDefault(
3641 MachineFunction &MF) const {
3642 return MF.getFunction().hasMinSize();
3643}
3644
3645static bool isCandidatePatchable(const MachineBasicBlock &MBB) {
3646 const MachineFunction *MF = MBB.getParent();
3647 const Function &F = MF->getFunction();
3648 return F.getFnAttribute(Kind: "fentry-call").getValueAsBool() ||
3649 F.hasFnAttribute(Kind: "patchable-function-entry");
3650}
3651
3652static bool isMIReadsReg(const MachineInstr &MI, const TargetRegisterInfo *TRI,
3653 MCRegister RegNo) {
3654 return MI.readsRegister(Reg: RegNo, TRI) ||
3655 MI.getDesc().hasImplicitUseOfPhysReg(Reg: RegNo);
3656}
3657
3658static bool isMIModifiesReg(const MachineInstr &MI,
3659 const TargetRegisterInfo *TRI, MCRegister RegNo) {
3660 return MI.modifiesRegister(Reg: RegNo, TRI) ||
3661 MI.getDesc().hasImplicitDefOfPhysReg(Reg: RegNo);
3662}
3663
3664static bool cannotInsertTailCall(const MachineBasicBlock &MBB) {
3665 if (!MBB.back().isReturn())
3666 return true;
3667 if (isCandidatePatchable(MBB))
3668 return true;
3669
3670 // If the candidate reads the pre-set register
3671 // that can be used for expanding PseudoTAIL instruction,
3672 // then we cannot insert tail call.
3673 const TargetSubtargetInfo &STI = MBB.getParent()->getSubtarget();
3674 MCRegister TailExpandUseRegNo =
3675 RISCVII::getTailExpandUseRegNo(FeatureBits: STI.getFeatureBits());
3676 for (const MachineInstr &MI : MBB) {
3677 if (isMIReadsReg(MI, TRI: STI.getRegisterInfo(), RegNo: TailExpandUseRegNo))
3678 return true;
3679 if (isMIModifiesReg(MI, TRI: STI.getRegisterInfo(), RegNo: TailExpandUseRegNo))
3680 break;
3681 }
3682 return false;
3683}
3684
3685bool RISCVInstrInfo::analyzeCandidate(outliner::Candidate &C) const {
3686 // If the expansion register for tail calls is live across the candidate
3687 // outlined call site, we cannot outline that candidate as the expansion
3688 // would clobber the register.
3689 MCRegister TailExpandUseReg =
3690 RISCVII::getTailExpandUseRegNo(FeatureBits: STI.getFeatureBits());
3691 if (C.back().isReturn() &&
3692 !C.isAvailableAcrossAndOutOfSeq(Reg: TailExpandUseReg, TRI: RegInfo)) {
3693 LLVM_DEBUG(dbgs() << "MBB:\n" << *C.getMBB());
3694 LLVM_DEBUG(dbgs() << "Cannot be outlined between: " << C.front() << "and "
3695 << C.back());
3696 LLVM_DEBUG(dbgs() << "Because the tail-call register is live across "
3697 "the proposed outlined function call\n");
3698 return true;
3699 }
3700
3701 // If last instruction is return then we can rely on
3702 // the verification already performed in the getOutliningTypeImpl.
3703 if (C.back().isReturn()) {
3704 assert(!cannotInsertTailCall(*C.getMBB()) &&
3705 "The candidate who uses return instruction must be outlined "
3706 "using tail call");
3707 return false;
3708 }
3709
3710 // Filter out candidates where the X5 register (t0) can't be used to setup
3711 // the function call.
3712 if (llvm::any_of(Range&: C, P: [this](const MachineInstr &MI) {
3713 return isMIModifiesReg(MI, TRI: &RegInfo, RegNo: RISCV::X5);
3714 }))
3715 return true;
3716
3717 return !C.isAvailableAcrossAndOutOfSeq(Reg: RISCV::X5, TRI: RegInfo);
3718}
3719
3720std::optional<std::unique_ptr<outliner::OutlinedFunction>>
3721RISCVInstrInfo::getOutliningCandidateInfo(
3722 const MachineModuleInfo &MMI,
3723 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
3724 unsigned MinRepeats) const {
3725
3726 // Analyze each candidate and erase the ones that are not viable.
3727 llvm::erase_if(C&: RepeatedSequenceLocs, P: [this](auto Candidate) {
3728 return analyzeCandidate(C&: Candidate);
3729 });
3730
3731 // If the sequence doesn't have enough candidates left, then we're done.
3732 if (RepeatedSequenceLocs.size() < MinRepeats)
3733 return std::nullopt;
3734
3735 // Each RepeatedSequenceLoc is identical.
3736 outliner::Candidate &Candidate = RepeatedSequenceLocs[0];
3737 unsigned InstrSizeCExt =
3738 Candidate.getMF()->getSubtarget<RISCVSubtarget>().hasStdExtZca() ? 2 : 4;
3739 unsigned CallOverhead = 0, FrameOverhead = 0;
3740
3741 // Count the number of CFI instructions in the candidate, if present.
3742 unsigned CFICount = 0;
3743 for (auto &I : Candidate) {
3744 if (I.isCFIInstruction())
3745 CFICount++;
3746 }
3747
3748 // Ensure CFI coverage matches: comparing the number of CFIs in the candidate
3749 // with the total number of CFIs in the parent function for each candidate.
3750 // Outlining only a subset of a function’s CFIs would split the unwind state
3751 // across two code regions and lead to incorrect address offsets between the
3752 // outlined body and the remaining code. To preserve correct unwind info, we
3753 // only outline when all CFIs in the function can be outlined together.
3754 for (outliner::Candidate &C : RepeatedSequenceLocs) {
3755 std::vector<MCCFIInstruction> CFIInstructions =
3756 C.getMF()->getFrameInstructions();
3757
3758 if (CFICount > 0 && CFICount != CFIInstructions.size())
3759 return std::nullopt;
3760 }
3761
3762 MachineOutlinerConstructionID MOCI = MachineOutlinerDefault;
3763 if (Candidate.back().isReturn()) {
3764 MOCI = MachineOutlinerTailCall;
3765 // tail call = auipc + jalr in the worst case without linker relaxation.
3766 // FIXME: This code suggests the JALR can be compressed - how?
3767 CallOverhead = 4 + InstrSizeCExt;
3768 // Using tail call we move ret instruction from caller to callee.
3769 FrameOverhead = 0;
3770 } else {
3771 // call t0, function = 8 bytes.
3772 CallOverhead = 8;
3773 // jr t0 = 4 bytes, 2 bytes if compressed instructions are enabled.
3774 FrameOverhead = InstrSizeCExt;
3775 }
3776
3777 // If we have CFI instructions, we can only outline if the outlined section
3778 // can be a tail call.
3779 if (MOCI != MachineOutlinerTailCall && CFICount > 0)
3780 return std::nullopt;
3781
3782 for (auto &C : RepeatedSequenceLocs)
3783 C.setCallInfo(CID: MOCI, CO: CallOverhead);
3784
3785 unsigned SequenceSize = 0;
3786 for (auto &MI : Candidate)
3787 SequenceSize += getInstSizeInBytes(MI);
3788
3789 return std::make_unique<outliner::OutlinedFunction>(
3790 args&: RepeatedSequenceLocs, args&: SequenceSize, args&: FrameOverhead, args&: MOCI);
3791}
3792
3793outliner::InstrType
3794RISCVInstrInfo::getOutliningTypeImpl(const MachineModuleInfo &MMI,
3795 MachineBasicBlock::iterator &MBBI,
3796 unsigned Flags) const {
3797 MachineInstr &MI = *MBBI;
3798 MachineBasicBlock *MBB = MI.getParent();
3799 const TargetRegisterInfo *TRI =
3800 MBB->getParent()->getSubtarget().getRegisterInfo();
3801 const auto &F = MI.getMF()->getFunction();
3802
3803 // We can only outline CFI instructions if we will tail call the outlined
3804 // function, or fix up the CFI offsets. Currently, CFI instructions are
3805 // outlined only if in a tail call.
3806 if (MI.isCFIInstruction())
3807 return outliner::InstrType::Legal;
3808
3809 if (cannotInsertTailCall(MBB: *MBB) &&
3810 (MI.isReturn() || isMIModifiesReg(MI, TRI, RegNo: RISCV::X5)))
3811 return outliner::InstrType::Illegal;
3812
3813 // Make sure the operands don't reference something unsafe.
3814 for (const auto &MO : MI.operands()) {
3815
3816 // pcrel-hi and pcrel-lo can't put in separate sections, filter that out
3817 // if any possible.
3818 if (MO.getTargetFlags() == RISCVII::MO_PCREL_LO &&
3819 (MI.getMF()->getTarget().getFunctionSections() || F.hasComdat() ||
3820 F.hasSection() || F.getSectionPrefix()))
3821 return outliner::InstrType::Illegal;
3822 }
3823
3824 if (isLPAD(MI))
3825 return outliner::InstrType::Illegal;
3826
3827 return outliner::InstrType::Legal;
3828}
3829
3830void RISCVInstrInfo::buildOutlinedFrame(
3831 MachineBasicBlock &MBB, MachineFunction &MF,
3832 const outliner::OutlinedFunction &OF) const {
3833
3834 if (OF.FrameConstructionID == MachineOutlinerTailCall)
3835 return;
3836
3837 MBB.addLiveIn(PhysReg: RISCV::X5);
3838
3839 // Add in a return instruction to the end of the outlined frame.
3840 MBB.insert(I: MBB.end(), MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: RISCV::JALR))
3841 .addReg(RegNo: RISCV::X0, Flags: RegState::Define)
3842 .addReg(RegNo: RISCV::X5)
3843 .addImm(Val: 0));
3844}
3845
3846MachineBasicBlock::iterator RISCVInstrInfo::insertOutlinedCall(
3847 Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It,
3848 MachineFunction &MF, outliner::Candidate &C) const {
3849
3850 if (C.CallConstructionID == MachineOutlinerTailCall) {
3851 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: RISCV::PseudoTAIL))
3852 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName()),
3853 /*Offset=*/0, TargetFlags: RISCVII::MO_CALL));
3854 return It;
3855 }
3856
3857 // Add in a call instruction to the outlined function at the given location.
3858 It = MBB.insert(I: It,
3859 MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: RISCV::PseudoCALLReg), DestReg: RISCV::X5)
3860 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName()), Offset: 0,
3861 TargetFlags: RISCVII::MO_CALL));
3862 return It;
3863}
3864
3865std::optional<RegImmPair> RISCVInstrInfo::isAddImmediate(const MachineInstr &MI,
3866 Register Reg) const {
3867 // TODO: Handle cases where Reg is a super- or sub-register of the
3868 // destination register.
3869 const MachineOperand &Op0 = MI.getOperand(i: 0);
3870 if (!Op0.isReg() || Reg != Op0.getReg())
3871 return std::nullopt;
3872
3873 // Don't consider ADDIW as a candidate because the caller may not be aware
3874 // of its sign extension behaviour.
3875 if (MI.getOpcode() == RISCV::ADDI && MI.getOperand(i: 1).isReg() &&
3876 MI.getOperand(i: 2).isImm())
3877 return RegImmPair{MI.getOperand(i: 1).getReg(), MI.getOperand(i: 2).getImm()};
3878
3879 return std::nullopt;
3880}
3881
3882// MIR printer helper function to annotate Operands with a comment.
3883std::string RISCVInstrInfo::createMIROperandComment(
3884 const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx,
3885 const TargetRegisterInfo *TRI) const {
3886 // Print a generic comment for this operand if there is one.
3887 std::string GenericComment =
3888 TargetInstrInfo::createMIROperandComment(MI, Op, OpIdx, TRI);
3889 if (!GenericComment.empty())
3890 return GenericComment;
3891
3892 const MCInstrDesc &Desc = MI.getDesc();
3893 if (OpIdx >= Desc.getNumOperands())
3894 return std::string();
3895
3896 std::string Comment;
3897 raw_string_ostream OS(Comment);
3898
3899 const MCOperandInfo &OpInfo = Desc.operands()[OpIdx];
3900
3901 // Print the full VType operand of vsetvli/vsetivli instructions, and the SEW
3902 // operand of vector codegen pseudos.
3903 switch (OpInfo.OperandType) {
3904 case RISCVOp::OPERAND_VTYPEI10:
3905 case RISCVOp::OPERAND_VTYPEI11: {
3906 unsigned Imm = Op.getImm();
3907 RISCVVType::printVType(VType: Imm, OS);
3908 break;
3909 }
3910 case RISCVOp::OPERAND_XSFMM_VTYPE: {
3911 unsigned Imm = Op.getImm();
3912 RISCVVType::printXSfmmVType(VType: Imm, OS);
3913 break;
3914 }
3915 case RISCVOp::OPERAND_XSFMM_TWIDEN: {
3916 unsigned Imm = Op.getImm();
3917 OS << "w" << Imm;
3918 break;
3919 }
3920 case RISCVOp::OPERAND_SEW:
3921 case RISCVOp::OPERAND_SEW_MASK: {
3922 unsigned Log2SEW = Op.getImm();
3923 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
3924 assert(RISCVVType::isValidSEW(SEW) && "Unexpected SEW");
3925 OS << "e" << SEW;
3926 break;
3927 }
3928 case RISCVOp::OPERAND_VEC_POLICY: {
3929 unsigned Policy = Op.getImm();
3930 assert(Policy <= (RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC) &&
3931 "Invalid Policy Value");
3932 OS << (Policy & RISCVVType::TAIL_AGNOSTIC ? "ta" : "tu") << ", "
3933 << (Policy & RISCVVType::MASK_AGNOSTIC ? "ma" : "mu");
3934 break;
3935 }
3936 case RISCVOp::OPERAND_AVL:
3937 if (Op.isImm() && Op.getImm() == -1)
3938 OS << "vl=VLMAX";
3939 else
3940 OS << "vl";
3941 break;
3942 case RISCVOp::OPERAND_VEC_RM:
3943 if (RISCVII::usesVXRM(TSFlags: Desc.TSFlags)) {
3944 assert(RISCVVXRndMode::isValidRoundingMode(Op.getImm()));
3945 auto VXRM = static_cast<RISCVVXRndMode::RoundingMode>(Op.getImm());
3946 OS << "vxrm=" << RISCVVXRndMode::roundingModeToString(RndMode: VXRM);
3947 } else {
3948 assert(RISCVFPRndMode::isValidRoundingMode(Op.getImm()));
3949 auto FRM = static_cast<RISCVFPRndMode::RoundingMode>(Op.getImm());
3950 OS << "frm=" << RISCVFPRndMode::roundingModeToString(RndMode: FRM);
3951 }
3952 break;
3953 }
3954
3955 return Comment;
3956}
3957
3958// clang-format off
3959#define CASE_RVV_OPCODE_UNMASK_LMUL(OP, LMUL) \
3960 RISCV::Pseudo##OP##_##LMUL
3961
3962#define CASE_RVV_OPCODE_MASK_LMUL(OP, LMUL) \
3963 RISCV::Pseudo##OP##_##LMUL##_MASK
3964
3965#define CASE_RVV_OPCODE_LMUL(OP, LMUL) \
3966 CASE_RVV_OPCODE_UNMASK_LMUL(OP, LMUL): \
3967 case CASE_RVV_OPCODE_MASK_LMUL(OP, LMUL)
3968
3969#define CASE_RVV_OPCODE_UNMASK_WIDEN(OP) \
3970 CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF8): \
3971 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF4): \
3972 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF2): \
3973 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M1): \
3974 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M2): \
3975 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M4)
3976
3977#define CASE_RVV_OPCODE_UNMASK(OP) \
3978 CASE_RVV_OPCODE_UNMASK_WIDEN(OP): \
3979 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M8)
3980
3981#define CASE_RVV_OPCODE_MASK_WIDEN(OP) \
3982 CASE_RVV_OPCODE_MASK_LMUL(OP, MF8): \
3983 case CASE_RVV_OPCODE_MASK_LMUL(OP, MF4): \
3984 case CASE_RVV_OPCODE_MASK_LMUL(OP, MF2): \
3985 case CASE_RVV_OPCODE_MASK_LMUL(OP, M1): \
3986 case CASE_RVV_OPCODE_MASK_LMUL(OP, M2): \
3987 case CASE_RVV_OPCODE_MASK_LMUL(OP, M4)
3988
3989#define CASE_RVV_OPCODE_MASK(OP) \
3990 CASE_RVV_OPCODE_MASK_WIDEN(OP): \
3991 case CASE_RVV_OPCODE_MASK_LMUL(OP, M8)
3992
3993#define CASE_RVV_OPCODE_WIDEN(OP) \
3994 CASE_RVV_OPCODE_UNMASK_WIDEN(OP): \
3995 case CASE_RVV_OPCODE_MASK_WIDEN(OP)
3996
3997#define CASE_RVV_OPCODE(OP) \
3998 CASE_RVV_OPCODE_UNMASK(OP): \
3999 case CASE_RVV_OPCODE_MASK(OP)
4000// clang-format on
4001
4002// clang-format off
4003#define CASE_VMA_OPCODE_COMMON(OP, TYPE, LMUL) \
4004 RISCV::PseudoV##OP##_##TYPE##_##LMUL
4005
4006#define CASE_VMA_OPCODE_LMULS(OP, TYPE) \
4007 CASE_VMA_OPCODE_COMMON(OP, TYPE, MF8): \
4008 case CASE_VMA_OPCODE_COMMON(OP, TYPE, MF4): \
4009 case CASE_VMA_OPCODE_COMMON(OP, TYPE, MF2): \
4010 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M1): \
4011 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M2): \
4012 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M4): \
4013 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M8)
4014
4015// VFMA instructions are SEW specific.
4016#define CASE_VFMA_OPCODE_COMMON(OP, TYPE, LMUL, SEW) \
4017 RISCV::PseudoV##OP##_##TYPE##_##LMUL##_##SEW
4018
4019#define CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE, SEW) \
4020 CASE_VFMA_OPCODE_COMMON(OP, TYPE, M1, SEW): \
4021 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M2, SEW): \
4022 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M4, SEW): \
4023 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M8, SEW)
4024
4025#define CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE, SEW) \
4026 CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF2, SEW): \
4027 case CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE, SEW)
4028
4029#define CASE_VFMA_OPCODE_LMULS_MF4(OP, TYPE, SEW) \
4030 CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF4, SEW): \
4031 case CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE, SEW)
4032
4033#define CASE_VFMA_OPCODE_VV(OP) \
4034 CASE_VFMA_OPCODE_LMULS_MF4(OP, VV, E16): \
4035 case CASE_VFMA_OPCODE_LMULS_MF4(OP##_ALT, VV, E16): \
4036 case CASE_VFMA_OPCODE_LMULS_MF2(OP, VV, E32): \
4037 case CASE_VFMA_OPCODE_LMULS_M1(OP, VV, E64)
4038
4039#define CASE_VFMA_SPLATS(OP) \
4040 CASE_VFMA_OPCODE_LMULS_MF4(OP, VFPR16, E16): \
4041 case CASE_VFMA_OPCODE_LMULS_MF4(OP##_ALT, VFPR16, E16): \
4042 case CASE_VFMA_OPCODE_LMULS_MF2(OP, VFPR32, E32): \
4043 case CASE_VFMA_OPCODE_LMULS_M1(OP, VFPR64, E64)
4044// clang-format on
4045
4046bool RISCVInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
4047 unsigned &SrcOpIdx1,
4048 unsigned &SrcOpIdx2) const {
4049 const MCInstrDesc &Desc = MI.getDesc();
4050 if (!Desc.isCommutable())
4051 return false;
4052
4053 switch (MI.getOpcode()) {
4054 case RISCV::TH_MVEQZ:
4055 case RISCV::TH_MVNEZ:
4056 // We can't commute operands if operand 2 (i.e., rs1 in
4057 // mveqz/mvnez rd,rs1,rs2) is the zero-register (as it is
4058 // not valid as the in/out-operand 1).
4059 if (MI.getOperand(i: 2).getReg() == RISCV::X0)
4060 return false;
4061 // Operands 1 and 2 are commutable, if we switch the opcode.
4062 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 1, CommutableOpIdx2: 2);
4063 case RISCV::QC_SELECTIEQ:
4064 case RISCV::QC_SELECTINE:
4065 case RISCV::QC_SELECTIIEQ:
4066 case RISCV::QC_SELECTIINE:
4067 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 1, CommutableOpIdx2: 2);
4068 case RISCV::QC_MVEQ:
4069 case RISCV::QC_MVNE:
4070 case RISCV::QC_MVLT:
4071 case RISCV::QC_MVGE:
4072 case RISCV::QC_MVLTU:
4073 case RISCV::QC_MVGEU:
4074 case RISCV::QC_MVEQI:
4075 case RISCV::QC_MVNEI:
4076 case RISCV::QC_MVLTI:
4077 case RISCV::QC_MVGEI:
4078 case RISCV::QC_MVLTUI:
4079 case RISCV::QC_MVGEUI:
4080 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 1, CommutableOpIdx2: 4);
4081 case RISCV::TH_MULA:
4082 case RISCV::TH_MULAW:
4083 case RISCV::TH_MULAH:
4084 case RISCV::TH_MULS:
4085 case RISCV::TH_MULSW:
4086 case RISCV::TH_MULSH:
4087 // Operands 2 and 3 are commutable.
4088 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 2, CommutableOpIdx2: 3);
4089 case RISCV::PseudoCCMOVGPRNoX0:
4090 case RISCV::PseudoCCMOVGPR:
4091 // Operands 1 and 2 are commutable.
4092 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 1, CommutableOpIdx2: 2);
4093 case CASE_RVV_OPCODE(VADD_VV):
4094 case CASE_RVV_OPCODE(VAND_VV):
4095 case CASE_RVV_OPCODE(VOR_VV):
4096 case CASE_RVV_OPCODE(VXOR_VV):
4097 case CASE_RVV_OPCODE_MASK(VMSEQ_VV):
4098 case CASE_RVV_OPCODE_MASK(VMSNE_VV):
4099 case CASE_RVV_OPCODE(VMIN_VV):
4100 case CASE_RVV_OPCODE(VMINU_VV):
4101 case CASE_RVV_OPCODE(VMAX_VV):
4102 case CASE_RVV_OPCODE(VMAXU_VV):
4103 case CASE_RVV_OPCODE(VMUL_VV):
4104 case CASE_RVV_OPCODE(VMULH_VV):
4105 case CASE_RVV_OPCODE(VMULHU_VV):
4106 case CASE_RVV_OPCODE_WIDEN(VWADD_VV):
4107 case CASE_RVV_OPCODE_WIDEN(VWADDU_VV):
4108 case CASE_RVV_OPCODE_WIDEN(VWMUL_VV):
4109 case CASE_RVV_OPCODE_WIDEN(VWMULU_VV):
4110 case CASE_RVV_OPCODE_WIDEN(VWMACC_VV):
4111 case CASE_RVV_OPCODE_WIDEN(VWMACCU_VV):
4112 case CASE_RVV_OPCODE(VABD_VV):
4113 case CASE_RVV_OPCODE(VABDU_VV):
4114 case CASE_RVV_OPCODE_WIDEN(VWABDA_VV):
4115 case CASE_RVV_OPCODE_WIDEN(VWABDAU_VV):
4116 case CASE_RVV_OPCODE_UNMASK(VADC_VVM):
4117 case CASE_RVV_OPCODE(VSADD_VV):
4118 case CASE_RVV_OPCODE(VSADDU_VV):
4119 case CASE_RVV_OPCODE(VAADD_VV):
4120 case CASE_RVV_OPCODE(VAADDU_VV):
4121 case CASE_RVV_OPCODE(VSMUL_VV):
4122 // Operands 2 and 3 are commutable.
4123 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 2, CommutableOpIdx2: 3);
4124 case CASE_VFMA_SPLATS(FMADD):
4125 case CASE_VFMA_SPLATS(FMSUB):
4126 case CASE_VFMA_SPLATS(FMACC):
4127 case CASE_VFMA_SPLATS(FMSAC):
4128 case CASE_VFMA_SPLATS(FNMADD):
4129 case CASE_VFMA_SPLATS(FNMSUB):
4130 case CASE_VFMA_SPLATS(FNMACC):
4131 case CASE_VFMA_SPLATS(FNMSAC):
4132 case CASE_VFMA_OPCODE_VV(FMACC):
4133 case CASE_VFMA_OPCODE_VV(FMSAC):
4134 case CASE_VFMA_OPCODE_VV(FNMACC):
4135 case CASE_VFMA_OPCODE_VV(FNMSAC):
4136 case CASE_VMA_OPCODE_LMULS(MADD, VX):
4137 case CASE_VMA_OPCODE_LMULS(NMSUB, VX):
4138 case CASE_VMA_OPCODE_LMULS(MACC, VX):
4139 case CASE_VMA_OPCODE_LMULS(NMSAC, VX):
4140 case CASE_VMA_OPCODE_LMULS(MACC, VV):
4141 case CASE_VMA_OPCODE_LMULS(NMSAC, VV): {
4142 // If the tail policy is undisturbed we can't commute.
4143 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags));
4144 if ((MI.getOperand(i: RISCVII::getVecPolicyOpNum(Desc: MI.getDesc())).getImm() &
4145 1) == 0)
4146 return false;
4147
4148 // For these instructions we can only swap operand 1 and operand 3 by
4149 // changing the opcode.
4150 unsigned CommutableOpIdx1 = 1;
4151 unsigned CommutableOpIdx2 = 3;
4152 if (!fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1,
4153 CommutableOpIdx2))
4154 return false;
4155 return true;
4156 }
4157 case CASE_VFMA_OPCODE_VV(FMADD):
4158 case CASE_VFMA_OPCODE_VV(FMSUB):
4159 case CASE_VFMA_OPCODE_VV(FNMADD):
4160 case CASE_VFMA_OPCODE_VV(FNMSUB):
4161 case CASE_VMA_OPCODE_LMULS(MADD, VV):
4162 case CASE_VMA_OPCODE_LMULS(NMSUB, VV): {
4163 // If the tail policy is undisturbed we can't commute.
4164 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags));
4165 if ((MI.getOperand(i: RISCVII::getVecPolicyOpNum(Desc: MI.getDesc())).getImm() &
4166 1) == 0)
4167 return false;
4168
4169 // For these instructions we have more freedom. We can commute with the
4170 // other multiplicand or with the addend/subtrahend/minuend.
4171
4172 // Any fixed operand must be from source 1, 2 or 3.
4173 if (SrcOpIdx1 != CommuteAnyOperandIndex && SrcOpIdx1 > 3)
4174 return false;
4175 if (SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx2 > 3)
4176 return false;
4177
4178 // It both ops are fixed one must be the tied source.
4179 if (SrcOpIdx1 != CommuteAnyOperandIndex &&
4180 SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx1 != 1 && SrcOpIdx2 != 1)
4181 return false;
4182
4183 // Look for two different register operands assumed to be commutable
4184 // regardless of the FMA opcode. The FMA opcode is adjusted later if
4185 // needed.
4186 if (SrcOpIdx1 == CommuteAnyOperandIndex ||
4187 SrcOpIdx2 == CommuteAnyOperandIndex) {
4188 // At least one of operands to be commuted is not specified and
4189 // this method is free to choose appropriate commutable operands.
4190 unsigned CommutableOpIdx1 = SrcOpIdx1;
4191 if (SrcOpIdx1 == SrcOpIdx2) {
4192 // Both of operands are not fixed. Set one of commutable
4193 // operands to the tied source.
4194 CommutableOpIdx1 = 1;
4195 } else if (SrcOpIdx1 == CommuteAnyOperandIndex) {
4196 // Only one of the operands is not fixed.
4197 CommutableOpIdx1 = SrcOpIdx2;
4198 }
4199
4200 // CommutableOpIdx1 is well defined now. Let's choose another commutable
4201 // operand and assign its index to CommutableOpIdx2.
4202 unsigned CommutableOpIdx2;
4203 if (CommutableOpIdx1 != 1) {
4204 // If we haven't already used the tied source, we must use it now.
4205 CommutableOpIdx2 = 1;
4206 } else {
4207 Register Op1Reg = MI.getOperand(i: CommutableOpIdx1).getReg();
4208
4209 // The commuted operands should have different registers.
4210 // Otherwise, the commute transformation does not change anything and
4211 // is useless. We use this as a hint to make our decision.
4212 if (Op1Reg != MI.getOperand(i: 2).getReg())
4213 CommutableOpIdx2 = 2;
4214 else
4215 CommutableOpIdx2 = 3;
4216 }
4217
4218 // Assign the found pair of commutable indices to SrcOpIdx1 and
4219 // SrcOpIdx2 to return those values.
4220 if (!fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1,
4221 CommutableOpIdx2))
4222 return false;
4223 }
4224
4225 return true;
4226 }
4227 }
4228
4229 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
4230}
4231
4232// clang-format off
4233#define CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL) \
4234 case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL: \
4235 Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL; \
4236 break;
4237
4238#define CASE_VMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE) \
4239 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF8) \
4240 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4) \
4241 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2) \
4242 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1) \
4243 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2) \
4244 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4) \
4245 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8)
4246
4247// VFMA depends on SEW.
4248#define CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL, SEW) \
4249 case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL##_##SEW: \
4250 Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL##_##SEW; \
4251 break;
4252
4253#define CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE, SEW) \
4254 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1, SEW) \
4255 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2, SEW) \
4256 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4, SEW) \
4257 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8, SEW)
4258
4259#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE, SEW) \
4260 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2, SEW) \
4261 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE, SEW)
4262
4263#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE, SEW) \
4264 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4, SEW) \
4265 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE, SEW)
4266
4267#define CASE_VFMA_CHANGE_OPCODE_VV(OLDOP, NEWOP) \
4268 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VV, E16) \
4269 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP##_ALT, NEWOP##_ALT, VV, E16) \
4270 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VV, E32) \
4271 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VV, E64)
4272
4273#define CASE_VFMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP) \
4274 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VFPR16, E16) \
4275 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP##_ALT, NEWOP##_ALT, VFPR16, E16) \
4276 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VFPR32, E32) \
4277 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VFPR64, E64)
4278// clang-format on
4279
4280MachineInstr *RISCVInstrInfo::commuteInstructionImpl(MachineInstr &MI,
4281 bool NewMI,
4282 unsigned OpIdx1,
4283 unsigned OpIdx2) const {
4284 auto cloneIfNew = [NewMI](MachineInstr &MI) -> MachineInstr & {
4285 if (NewMI)
4286 return *MI.getParent()->getParent()->CloneMachineInstr(Orig: &MI);
4287 return MI;
4288 };
4289
4290 switch (MI.getOpcode()) {
4291 case RISCV::TH_MVEQZ:
4292 case RISCV::TH_MVNEZ: {
4293 auto &WorkingMI = cloneIfNew(MI);
4294 WorkingMI.setDesc(get(Opcode: MI.getOpcode() == RISCV::TH_MVEQZ ? RISCV::TH_MVNEZ
4295 : RISCV::TH_MVEQZ));
4296 return TargetInstrInfo::commuteInstructionImpl(MI&: WorkingMI, NewMI: false, OpIdx1,
4297 OpIdx2);
4298 }
4299 case RISCV::QC_SELECTIEQ:
4300 case RISCV::QC_SELECTINE:
4301 case RISCV::QC_SELECTIIEQ:
4302 case RISCV::QC_SELECTIINE:
4303 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
4304 case RISCV::QC_MVEQ:
4305 case RISCV::QC_MVNE:
4306 case RISCV::QC_MVLT:
4307 case RISCV::QC_MVGE:
4308 case RISCV::QC_MVLTU:
4309 case RISCV::QC_MVGEU:
4310 case RISCV::QC_MVEQI:
4311 case RISCV::QC_MVNEI:
4312 case RISCV::QC_MVLTI:
4313 case RISCV::QC_MVGEI:
4314 case RISCV::QC_MVLTUI:
4315 case RISCV::QC_MVGEUI: {
4316 auto &WorkingMI = cloneIfNew(MI);
4317 WorkingMI.setDesc(get(Opcode: getInverseXqcicmOpcode(Opcode: MI.getOpcode())));
4318 return TargetInstrInfo::commuteInstructionImpl(MI&: WorkingMI, NewMI: false, OpIdx1,
4319 OpIdx2);
4320 }
4321 case RISCV::PseudoCCMOVGPRNoX0:
4322 case RISCV::PseudoCCMOVGPR: {
4323 // CCMOV can be commuted by inverting the condition.
4324 unsigned BCC = MI.getOperand(i: MI.getNumExplicitOperands() - 3).getImm();
4325 BCC = RISCVCC::getInverseBranchOpcode(BCC);
4326 auto &WorkingMI = cloneIfNew(MI);
4327 WorkingMI.getOperand(i: MI.getNumExplicitOperands() - 3).setImm(BCC);
4328 return TargetInstrInfo::commuteInstructionImpl(MI&: WorkingMI, /*NewMI*/ false,
4329 OpIdx1, OpIdx2);
4330 }
4331 case CASE_VFMA_SPLATS(FMACC):
4332 case CASE_VFMA_SPLATS(FMADD):
4333 case CASE_VFMA_SPLATS(FMSAC):
4334 case CASE_VFMA_SPLATS(FMSUB):
4335 case CASE_VFMA_SPLATS(FNMACC):
4336 case CASE_VFMA_SPLATS(FNMADD):
4337 case CASE_VFMA_SPLATS(FNMSAC):
4338 case CASE_VFMA_SPLATS(FNMSUB):
4339 case CASE_VFMA_OPCODE_VV(FMACC):
4340 case CASE_VFMA_OPCODE_VV(FMSAC):
4341 case CASE_VFMA_OPCODE_VV(FNMACC):
4342 case CASE_VFMA_OPCODE_VV(FNMSAC):
4343 case CASE_VMA_OPCODE_LMULS(MADD, VX):
4344 case CASE_VMA_OPCODE_LMULS(NMSUB, VX):
4345 case CASE_VMA_OPCODE_LMULS(MACC, VX):
4346 case CASE_VMA_OPCODE_LMULS(NMSAC, VX):
4347 case CASE_VMA_OPCODE_LMULS(MACC, VV):
4348 case CASE_VMA_OPCODE_LMULS(NMSAC, VV): {
4349 // It only make sense to toggle these between clobbering the
4350 // addend/subtrahend/minuend one of the multiplicands.
4351 assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");
4352 assert((OpIdx1 == 3 || OpIdx2 == 3) && "Unexpected opcode index");
4353 unsigned Opc;
4354 switch (MI.getOpcode()) {
4355 default:
4356 llvm_unreachable("Unexpected opcode");
4357 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMACC, FMADD)
4358 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMADD, FMACC)
4359 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMSAC, FMSUB)
4360 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMSUB, FMSAC)
4361 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMACC, FNMADD)
4362 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMADD, FNMACC)
4363 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMSAC, FNMSUB)
4364 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMSUB, FNMSAC)
4365 CASE_VFMA_CHANGE_OPCODE_VV(FMACC, FMADD)
4366 CASE_VFMA_CHANGE_OPCODE_VV(FMSAC, FMSUB)
4367 CASE_VFMA_CHANGE_OPCODE_VV(FNMACC, FNMADD)
4368 CASE_VFMA_CHANGE_OPCODE_VV(FNMSAC, FNMSUB)
4369 CASE_VMA_CHANGE_OPCODE_LMULS(MACC, MADD, VX)
4370 CASE_VMA_CHANGE_OPCODE_LMULS(MADD, MACC, VX)
4371 CASE_VMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VX)
4372 CASE_VMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VX)
4373 CASE_VMA_CHANGE_OPCODE_LMULS(MACC, MADD, VV)
4374 CASE_VMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VV)
4375 }
4376
4377 auto &WorkingMI = cloneIfNew(MI);
4378 WorkingMI.setDesc(get(Opcode: Opc));
4379 return TargetInstrInfo::commuteInstructionImpl(MI&: WorkingMI, /*NewMI=*/false,
4380 OpIdx1, OpIdx2);
4381 }
4382 case CASE_VFMA_OPCODE_VV(FMADD):
4383 case CASE_VFMA_OPCODE_VV(FMSUB):
4384 case CASE_VFMA_OPCODE_VV(FNMADD):
4385 case CASE_VFMA_OPCODE_VV(FNMSUB):
4386 case CASE_VMA_OPCODE_LMULS(MADD, VV):
4387 case CASE_VMA_OPCODE_LMULS(NMSUB, VV): {
4388 assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");
4389 // If one of the operands, is the addend we need to change opcode.
4390 // Otherwise we're just swapping 2 of the multiplicands.
4391 if (OpIdx1 == 3 || OpIdx2 == 3) {
4392 unsigned Opc;
4393 switch (MI.getOpcode()) {
4394 default:
4395 llvm_unreachable("Unexpected opcode");
4396 CASE_VFMA_CHANGE_OPCODE_VV(FMADD, FMACC)
4397 CASE_VFMA_CHANGE_OPCODE_VV(FMSUB, FMSAC)
4398 CASE_VFMA_CHANGE_OPCODE_VV(FNMADD, FNMACC)
4399 CASE_VFMA_CHANGE_OPCODE_VV(FNMSUB, FNMSAC)
4400 CASE_VMA_CHANGE_OPCODE_LMULS(MADD, MACC, VV)
4401 CASE_VMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VV)
4402 }
4403
4404 auto &WorkingMI = cloneIfNew(MI);
4405 WorkingMI.setDesc(get(Opcode: Opc));
4406 return TargetInstrInfo::commuteInstructionImpl(MI&: WorkingMI, /*NewMI=*/false,
4407 OpIdx1, OpIdx2);
4408 }
4409 // Let the default code handle it.
4410 break;
4411 }
4412 }
4413
4414 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
4415}
4416
4417#undef CASE_VMA_CHANGE_OPCODE_COMMON
4418#undef CASE_VMA_CHANGE_OPCODE_LMULS
4419#undef CASE_VFMA_CHANGE_OPCODE_COMMON
4420#undef CASE_VFMA_CHANGE_OPCODE_LMULS_M1
4421#undef CASE_VFMA_CHANGE_OPCODE_LMULS_MF2
4422#undef CASE_VFMA_CHANGE_OPCODE_LMULS_MF4
4423#undef CASE_VFMA_CHANGE_OPCODE_VV
4424#undef CASE_VFMA_CHANGE_OPCODE_SPLATS
4425
4426#undef CASE_RVV_OPCODE_UNMASK_LMUL
4427#undef CASE_RVV_OPCODE_MASK_LMUL
4428#undef CASE_RVV_OPCODE_LMUL
4429#undef CASE_RVV_OPCODE_UNMASK_WIDEN
4430#undef CASE_RVV_OPCODE_UNMASK
4431#undef CASE_RVV_OPCODE_MASK_WIDEN
4432#undef CASE_RVV_OPCODE_MASK
4433#undef CASE_RVV_OPCODE_WIDEN
4434#undef CASE_RVV_OPCODE
4435
4436#undef CASE_VMA_OPCODE_COMMON
4437#undef CASE_VMA_OPCODE_LMULS
4438#undef CASE_VFMA_OPCODE_COMMON
4439#undef CASE_VFMA_OPCODE_LMULS_M1
4440#undef CASE_VFMA_OPCODE_LMULS_MF2
4441#undef CASE_VFMA_OPCODE_LMULS_MF4
4442#undef CASE_VFMA_OPCODE_VV
4443#undef CASE_VFMA_SPLATS
4444
4445bool RISCVInstrInfo::simplifyInstruction(MachineInstr &MI) const {
4446 switch (MI.getOpcode()) {
4447 default:
4448 break;
4449 case RISCV::ADD:
4450 case RISCV::OR:
4451 case RISCV::XOR:
4452 // Normalize (so we hit the next if clause).
4453 // add/[x]or rd, zero, rs => add/[x]or rd, rs, zero
4454 if (MI.getOperand(i: 1).getReg() == RISCV::X0)
4455 commuteInstruction(MI);
4456 // add/[x]or rd, rs, zero => addi rd, rs, 0
4457 if (MI.getOperand(i: 2).getReg() == RISCV::X0) {
4458 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4459 MI.setDesc(get(Opcode: RISCV::ADDI));
4460 return true;
4461 }
4462 // xor rd, rs, rs => addi rd, zero, 0
4463 if (MI.getOpcode() == RISCV::XOR &&
4464 MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg()) {
4465 MI.getOperand(i: 1).setReg(RISCV::X0);
4466 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4467 MI.setDesc(get(Opcode: RISCV::ADDI));
4468 return true;
4469 }
4470 break;
4471 case RISCV::ORI:
4472 case RISCV::XORI:
4473 // [x]ori rd, zero, N => addi rd, zero, N
4474 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4475 MI.setDesc(get(Opcode: RISCV::ADDI));
4476 return true;
4477 }
4478 break;
4479 case RISCV::SUB:
4480 // sub rd, rs, zero => addi rd, rs, 0
4481 if (MI.getOperand(i: 2).getReg() == RISCV::X0) {
4482 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4483 MI.setDesc(get(Opcode: RISCV::ADDI));
4484 return true;
4485 }
4486 break;
4487 case RISCV::SUBW:
4488 // subw rd, rs, zero => addiw rd, rs, 0
4489 if (MI.getOperand(i: 2).getReg() == RISCV::X0) {
4490 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4491 MI.setDesc(get(Opcode: RISCV::ADDIW));
4492 return true;
4493 }
4494 break;
4495 case RISCV::ADDW:
4496 // Normalize (so we hit the next if clause).
4497 // addw rd, zero, rs => addw rd, rs, zero
4498 if (MI.getOperand(i: 1).getReg() == RISCV::X0)
4499 commuteInstruction(MI);
4500 // addw rd, rs, zero => addiw rd, rs, 0
4501 if (MI.getOperand(i: 2).getReg() == RISCV::X0) {
4502 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4503 MI.setDesc(get(Opcode: RISCV::ADDIW));
4504 return true;
4505 }
4506 break;
4507 case RISCV::SH1ADD:
4508 case RISCV::SH1ADD_UW:
4509 case RISCV::SH2ADD:
4510 case RISCV::SH2ADD_UW:
4511 case RISCV::SH3ADD:
4512 case RISCV::SH3ADD_UW:
4513 // shNadd[.uw] rd, zero, rs => addi rd, rs, 0
4514 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4515 MI.removeOperand(OpNo: 1);
4516 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
4517 MI.setDesc(get(Opcode: RISCV::ADDI));
4518 return true;
4519 }
4520 // shNadd[.uw] rd, rs, zero => slli[.uw] rd, rs, N
4521 if (MI.getOperand(i: 2).getReg() == RISCV::X0) {
4522 MI.removeOperand(OpNo: 2);
4523 unsigned Opc = MI.getOpcode();
4524 if (Opc == RISCV::SH1ADD_UW || Opc == RISCV::SH2ADD_UW ||
4525 Opc == RISCV::SH3ADD_UW) {
4526 MI.addOperand(Op: MachineOperand::CreateImm(Val: getSHXADDUWShiftAmount(Opc)));
4527 MI.setDesc(get(Opcode: RISCV::SLLI_UW));
4528 return true;
4529 }
4530 MI.addOperand(Op: MachineOperand::CreateImm(Val: getSHXADDShiftAmount(Opc)));
4531 MI.setDesc(get(Opcode: RISCV::SLLI));
4532 return true;
4533 }
4534 break;
4535 case RISCV::AND:
4536 case RISCV::MUL:
4537 case RISCV::MULH:
4538 case RISCV::MULHSU:
4539 case RISCV::MULHU:
4540 case RISCV::MULW:
4541 // and rd, zero, rs => addi rd, zero, 0
4542 // mul* rd, zero, rs => addi rd, zero, 0
4543 // and rd, rs, zero => addi rd, zero, 0
4544 // mul* rd, rs, zero => addi rd, zero, 0
4545 if (MI.getOperand(i: 1).getReg() == RISCV::X0 ||
4546 MI.getOperand(i: 2).getReg() == RISCV::X0) {
4547 MI.getOperand(i: 1).setReg(RISCV::X0);
4548 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4549 MI.setDesc(get(Opcode: RISCV::ADDI));
4550 return true;
4551 }
4552 break;
4553 case RISCV::ANDI:
4554 // andi rd, zero, C => addi rd, zero, 0
4555 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4556 MI.getOperand(i: 2).setImm(0);
4557 MI.setDesc(get(Opcode: RISCV::ADDI));
4558 return true;
4559 }
4560 break;
4561 case RISCV::SLL:
4562 case RISCV::SRL:
4563 case RISCV::SRA:
4564 // shift rd, zero, rs => addi rd, zero, 0
4565 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4566 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4567 MI.setDesc(get(Opcode: RISCV::ADDI));
4568 return true;
4569 }
4570 // shift rd, rs, zero => addi rd, rs, 0
4571 if (MI.getOperand(i: 2).getReg() == RISCV::X0) {
4572 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4573 MI.setDesc(get(Opcode: RISCV::ADDI));
4574 return true;
4575 }
4576 break;
4577 case RISCV::SLLW:
4578 case RISCV::SRLW:
4579 case RISCV::SRAW:
4580 // shiftw rd, zero, rs => addi rd, zero, 0
4581 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4582 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4583 MI.setDesc(get(Opcode: RISCV::ADDI));
4584 return true;
4585 }
4586 break;
4587 case RISCV::SLLI:
4588 case RISCV::SRLI:
4589 case RISCV::SRAI:
4590 case RISCV::SLLIW:
4591 case RISCV::SRLIW:
4592 case RISCV::SRAIW:
4593 case RISCV::SLLI_UW:
4594 // shiftimm rd, zero, N => addi rd, zero, 0
4595 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4596 MI.getOperand(i: 2).setImm(0);
4597 MI.setDesc(get(Opcode: RISCV::ADDI));
4598 return true;
4599 }
4600 break;
4601 case RISCV::SLTU:
4602 case RISCV::ADD_UW:
4603 // sltu rd, zero, zero => addi rd, zero, 0
4604 // add.uw rd, zero, zero => addi rd, zero, 0
4605 if (MI.getOperand(i: 1).getReg() == RISCV::X0 &&
4606 MI.getOperand(i: 2).getReg() == RISCV::X0) {
4607 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4608 MI.setDesc(get(Opcode: RISCV::ADDI));
4609 return true;
4610 }
4611 // add.uw rd, zero, rs => addi rd, rs, 0
4612 if (MI.getOpcode() == RISCV::ADD_UW &&
4613 MI.getOperand(i: 1).getReg() == RISCV::X0) {
4614 MI.removeOperand(OpNo: 1);
4615 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
4616 MI.setDesc(get(Opcode: RISCV::ADDI));
4617 }
4618 break;
4619 case RISCV::SLTIU:
4620 // sltiu rd, zero, NZC => addi rd, zero, 1
4621 // sltiu rd, zero, 0 => addi rd, zero, 0
4622 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4623 MI.getOperand(i: 2).setImm(MI.getOperand(i: 2).getImm() != 0);
4624 MI.setDesc(get(Opcode: RISCV::ADDI));
4625 return true;
4626 }
4627 break;
4628 case RISCV::SEXT_H:
4629 case RISCV::SEXT_B:
4630 case RISCV::ZEXT_H_RV32:
4631 case RISCV::ZEXT_H_RV64:
4632 // sext.[hb] rd, zero => addi rd, zero, 0
4633 // zext.h rd, zero => addi rd, zero, 0
4634 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4635 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
4636 MI.setDesc(get(Opcode: RISCV::ADDI));
4637 return true;
4638 }
4639 break;
4640 case RISCV::MIN:
4641 case RISCV::MINU:
4642 case RISCV::MAX:
4643 case RISCV::MAXU:
4644 // min|max rd, rs, rs => addi rd, rs, 0
4645 if (MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg()) {
4646 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4647 MI.setDesc(get(Opcode: RISCV::ADDI));
4648 return true;
4649 }
4650 break;
4651 case RISCV::BEQ:
4652 case RISCV::BNE:
4653 // b{eq,ne} zero, rs, imm => b{eq,ne} rs, zero, imm
4654 if (MI.getOperand(i: 0).getReg() == RISCV::X0) {
4655 MachineOperand MO0 = MI.getOperand(i: 0);
4656 MI.removeOperand(OpNo: 0);
4657 MI.insert(InsertBefore: MI.operands_begin() + 1, Ops: {MO0});
4658 }
4659 break;
4660 case RISCV::BLTU:
4661 // bltu zero, rs, imm => bne rs, zero, imm
4662 if (MI.getOperand(i: 0).getReg() == RISCV::X0) {
4663 MachineOperand MO0 = MI.getOperand(i: 0);
4664 MI.removeOperand(OpNo: 0);
4665 MI.insert(InsertBefore: MI.operands_begin() + 1, Ops: {MO0});
4666 MI.setDesc(get(Opcode: RISCV::BNE));
4667 }
4668 break;
4669 case RISCV::BGEU:
4670 // bgeu zero, rs, imm => beq rs, zero, imm
4671 if (MI.getOperand(i: 0).getReg() == RISCV::X0) {
4672 MachineOperand MO0 = MI.getOperand(i: 0);
4673 MI.removeOperand(OpNo: 0);
4674 MI.insert(InsertBefore: MI.operands_begin() + 1, Ops: {MO0});
4675 MI.setDesc(get(Opcode: RISCV::BEQ));
4676 }
4677 break;
4678 }
4679 return false;
4680}
4681
4682// clang-format off
4683#define CASE_WIDEOP_OPCODE_COMMON(OP, LMUL) \
4684 RISCV::PseudoV##OP##_##LMUL##_TIED
4685
4686#define CASE_WIDEOP_OPCODE_LMULS(OP) \
4687 CASE_WIDEOP_OPCODE_COMMON(OP, MF8): \
4688 case CASE_WIDEOP_OPCODE_COMMON(OP, MF4): \
4689 case CASE_WIDEOP_OPCODE_COMMON(OP, MF2): \
4690 case CASE_WIDEOP_OPCODE_COMMON(OP, M1): \
4691 case CASE_WIDEOP_OPCODE_COMMON(OP, M2): \
4692 case CASE_WIDEOP_OPCODE_COMMON(OP, M4)
4693
4694#define CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL) \
4695 case RISCV::PseudoV##OP##_##LMUL##_TIED: \
4696 NewOpc = RISCV::PseudoV##OP##_##LMUL; \
4697 break;
4698
4699#define CASE_WIDEOP_CHANGE_OPCODE_LMULS(OP) \
4700 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF8) \
4701 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4) \
4702 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2) \
4703 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1) \
4704 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2) \
4705 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4)
4706
4707// FP Widening Ops may by SEW aware. Create SEW aware cases for these cases.
4708#define CASE_FP_WIDEOP_OPCODE_COMMON(OP, LMUL, SEW) \
4709 RISCV::PseudoV##OP##_##LMUL##_##SEW##_TIED
4710
4711#define CASE_FP_WIDEOP_OPCODE_LMULS(OP) \
4712 CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF4, E16): \
4713 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E16): \
4714 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E32): \
4715 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E16): \
4716 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E32): \
4717 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E16): \
4718 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E32): \
4719 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E16): \
4720 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E32) \
4721
4722#define CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL, SEW) \
4723 case RISCV::PseudoV##OP##_##LMUL##_##SEW##_TIED: \
4724 NewOpc = RISCV::PseudoV##OP##_##LMUL##_##SEW; \
4725 break;
4726
4727#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(OP) \
4728 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4, E16) \
4729 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E16) \
4730 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E32) \
4731 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E16) \
4732 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E32) \
4733 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E16) \
4734 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E32) \
4735 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E16) \
4736 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E32) \
4737
4738#define CASE_FP_WIDEOP_OPCODE_LMULS_ALT(OP) \
4739 CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF4, E16): \
4740 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E16): \
4741 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E16): \
4742 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E16): \
4743 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E16)
4744
4745#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_ALT(OP) \
4746 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4, E16) \
4747 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E16) \
4748 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E16) \
4749 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E16) \
4750 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E16)
4751// clang-format on
4752
4753MachineInstr *RISCVInstrInfo::convertToThreeAddress(MachineInstr &MI,
4754 LiveVariables *LV,
4755 LiveIntervals *LIS) const {
4756 MachineInstrBuilder MIB;
4757 switch (MI.getOpcode()) {
4758 default:
4759 return nullptr;
4760 case CASE_FP_WIDEOP_OPCODE_LMULS_ALT(FWADD_ALT_WV):
4761 case CASE_FP_WIDEOP_OPCODE_LMULS_ALT(FWSUB_ALT_WV):
4762 case CASE_FP_WIDEOP_OPCODE_LMULS(FWADD_WV):
4763 case CASE_FP_WIDEOP_OPCODE_LMULS(FWSUB_WV): {
4764 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags) &&
4765 MI.getNumExplicitOperands() == 7 &&
4766 "Expect 7 explicit operands rd, rs2, rs1, rm, vl, sew, policy");
4767 // If the tail policy is undisturbed we can't convert.
4768 if ((MI.getOperand(i: RISCVII::getVecPolicyOpNum(Desc: MI.getDesc())).getImm() &
4769 1) == 0)
4770 return nullptr;
4771 // clang-format off
4772 unsigned NewOpc;
4773 switch (MI.getOpcode()) {
4774 default:
4775 llvm_unreachable("Unexpected opcode");
4776 CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(FWADD_WV)
4777 CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(FWSUB_WV)
4778 CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_ALT(FWADD_ALT_WV)
4779 CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_ALT(FWSUB_ALT_WV)
4780 }
4781 // clang-format on
4782
4783 MachineBasicBlock &MBB = *MI.getParent();
4784 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4785 .add(MO: MI.getOperand(i: 0))
4786 .addReg(RegNo: MI.getOperand(i: 0).getReg(), Flags: RegState::Undef)
4787 .add(MO: MI.getOperand(i: 1))
4788 .add(MO: MI.getOperand(i: 2))
4789 .add(MO: MI.getOperand(i: 3))
4790 .add(MO: MI.getOperand(i: 4))
4791 .add(MO: MI.getOperand(i: 5))
4792 .add(MO: MI.getOperand(i: 6));
4793 break;
4794 }
4795 case CASE_WIDEOP_OPCODE_LMULS(WADD_WV):
4796 case CASE_WIDEOP_OPCODE_LMULS(WADDU_WV):
4797 case CASE_WIDEOP_OPCODE_LMULS(WSUB_WV):
4798 case CASE_WIDEOP_OPCODE_LMULS(WSUBU_WV): {
4799 // If the tail policy is undisturbed we can't convert.
4800 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags) &&
4801 MI.getNumExplicitOperands() == 6);
4802 if ((MI.getOperand(i: RISCVII::getVecPolicyOpNum(Desc: MI.getDesc())).getImm() &
4803 1) == 0)
4804 return nullptr;
4805
4806 // clang-format off
4807 unsigned NewOpc;
4808 switch (MI.getOpcode()) {
4809 default:
4810 llvm_unreachable("Unexpected opcode");
4811 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WADD_WV)
4812 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WADDU_WV)
4813 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WSUB_WV)
4814 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WSUBU_WV)
4815 }
4816 // clang-format on
4817
4818 MachineBasicBlock &MBB = *MI.getParent();
4819 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4820 .add(MO: MI.getOperand(i: 0))
4821 .addReg(RegNo: MI.getOperand(i: 0).getReg(), Flags: RegState::Undef)
4822 .add(MO: MI.getOperand(i: 1))
4823 .add(MO: MI.getOperand(i: 2))
4824 .add(MO: MI.getOperand(i: 3))
4825 .add(MO: MI.getOperand(i: 4))
4826 .add(MO: MI.getOperand(i: 5));
4827 break;
4828 }
4829 }
4830 MIB.copyImplicitOps(OtherMI: MI);
4831
4832 if (LV) {
4833 unsigned NumOps = MI.getNumOperands();
4834 for (unsigned I = 1; I < NumOps; ++I) {
4835 MachineOperand &Op = MI.getOperand(i: I);
4836 if (Op.isReg() && Op.isKill())
4837 LV->replaceKillInstruction(Reg: Op.getReg(), OldMI&: MI, NewMI&: *MIB);
4838 }
4839 }
4840
4841 if (LIS) {
4842 SlotIndex Idx = LIS->ReplaceMachineInstrInMaps(MI, NewMI&: *MIB);
4843
4844 if (MI.getOperand(i: 0).isEarlyClobber()) {
4845 // Use operand 1 was tied to early-clobber def operand 0, so its live
4846 // interval could have ended at an early-clobber slot. Now they are not
4847 // tied we need to update it to the normal register slot.
4848 LiveInterval &LI = LIS->getInterval(Reg: MI.getOperand(i: 1).getReg());
4849 LiveRange::Segment *S = LI.getSegmentContaining(Idx);
4850 if (S->end == Idx.getRegSlot(EC: true))
4851 S->end = Idx.getRegSlot();
4852 }
4853 }
4854
4855 return MIB;
4856}
4857
4858#undef CASE_WIDEOP_OPCODE_COMMON
4859#undef CASE_WIDEOP_OPCODE_LMULS
4860#undef CASE_WIDEOP_CHANGE_OPCODE_COMMON
4861#undef CASE_WIDEOP_CHANGE_OPCODE_LMULS
4862#undef CASE_FP_WIDEOP_OPCODE_COMMON
4863#undef CASE_FP_WIDEOP_OPCODE_LMULS
4864#undef CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON
4865#undef CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS
4866
4867void RISCVInstrInfo::mulImm(MachineFunction &MF, MachineBasicBlock &MBB,
4868 MachineBasicBlock::iterator II, const DebugLoc &DL,
4869 Register DestReg, uint32_t Amount,
4870 MachineInstr::MIFlag Flag) const {
4871 MachineRegisterInfo &MRI = MF.getRegInfo();
4872 if (llvm::has_single_bit<uint32_t>(Value: Amount)) {
4873 uint32_t ShiftAmount = Log2_32(Value: Amount);
4874 if (ShiftAmount == 0)
4875 return;
4876 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::SLLI), DestReg)
4877 .addReg(RegNo: DestReg, Flags: RegState::Kill)
4878 .addImm(Val: ShiftAmount)
4879 .setMIFlag(Flag);
4880 } else if (int ShXAmount, ShiftAmount;
4881 STI.hasShlAdd(ShAmt: 3) &&
4882 (ShXAmount = isShifted359(Value: Amount, Shift&: ShiftAmount)) != 0) {
4883 // We can use Zba SHXADD+SLLI instructions for multiply in some cases.
4884 unsigned Opc;
4885 switch (ShXAmount) {
4886 case 1:
4887 Opc = RISCV::SH1ADD;
4888 break;
4889 case 2:
4890 Opc = RISCV::SH2ADD;
4891 break;
4892 case 3:
4893 Opc = RISCV::SH3ADD;
4894 break;
4895 default:
4896 llvm_unreachable("unexpected result of isShifted359");
4897 }
4898 if (ShiftAmount)
4899 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::SLLI), DestReg)
4900 .addReg(RegNo: DestReg, Flags: RegState::Kill)
4901 .addImm(Val: ShiftAmount)
4902 .setMIFlag(Flag);
4903 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: Opc), DestReg)
4904 .addReg(RegNo: DestReg, Flags: RegState::Kill)
4905 .addReg(RegNo: DestReg)
4906 .setMIFlag(Flag);
4907 } else if (llvm::has_single_bit<uint32_t>(Value: Amount - 1)) {
4908 Register ScaledRegister = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
4909 uint32_t ShiftAmount = Log2_32(Value: Amount - 1);
4910 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::SLLI), DestReg: ScaledRegister)
4911 .addReg(RegNo: DestReg)
4912 .addImm(Val: ShiftAmount)
4913 .setMIFlag(Flag);
4914 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::ADD), DestReg)
4915 .addReg(RegNo: ScaledRegister, Flags: RegState::Kill)
4916 .addReg(RegNo: DestReg, Flags: RegState::Kill)
4917 .setMIFlag(Flag);
4918 } else if (llvm::has_single_bit<uint32_t>(Value: Amount + 1)) {
4919 Register ScaledRegister = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
4920 uint32_t ShiftAmount = Log2_32(Value: Amount + 1);
4921 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::SLLI), DestReg: ScaledRegister)
4922 .addReg(RegNo: DestReg)
4923 .addImm(Val: ShiftAmount)
4924 .setMIFlag(Flag);
4925 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::SUB), DestReg)
4926 .addReg(RegNo: ScaledRegister, Flags: RegState::Kill)
4927 .addReg(RegNo: DestReg, Flags: RegState::Kill)
4928 .setMIFlag(Flag);
4929 } else if (STI.hasStdExtZmmul()) {
4930 Register N = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
4931 movImm(MBB, MBBI: II, DL, DstReg: N, Val: Amount, Flag);
4932 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::MUL), DestReg)
4933 .addReg(RegNo: DestReg, Flags: RegState::Kill)
4934 .addReg(RegNo: N, Flags: RegState::Kill)
4935 .setMIFlag(Flag);
4936 } else {
4937 Register Acc;
4938 uint32_t PrevShiftAmount = 0;
4939 for (uint32_t ShiftAmount = 0; Amount >> ShiftAmount; ShiftAmount++) {
4940 if (Amount & (1U << ShiftAmount)) {
4941 if (ShiftAmount)
4942 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::SLLI), DestReg)
4943 .addReg(RegNo: DestReg, Flags: RegState::Kill)
4944 .addImm(Val: ShiftAmount - PrevShiftAmount)
4945 .setMIFlag(Flag);
4946 if (Amount >> (ShiftAmount + 1)) {
4947 // If we don't have an accmulator yet, create it and copy DestReg.
4948 if (!Acc) {
4949 Acc = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
4950 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: TargetOpcode::COPY), DestReg: Acc)
4951 .addReg(RegNo: DestReg)
4952 .setMIFlag(Flag);
4953 } else {
4954 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::ADD), DestReg: Acc)
4955 .addReg(RegNo: Acc, Flags: RegState::Kill)
4956 .addReg(RegNo: DestReg)
4957 .setMIFlag(Flag);
4958 }
4959 }
4960 PrevShiftAmount = ShiftAmount;
4961 }
4962 }
4963 assert(Acc && "Expected valid accumulator");
4964 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::ADD), DestReg)
4965 .addReg(RegNo: DestReg, Flags: RegState::Kill)
4966 .addReg(RegNo: Acc, Flags: RegState::Kill)
4967 .setMIFlag(Flag);
4968 }
4969}
4970
4971ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
4972RISCVInstrInfo::getSerializableMachineMemOperandTargetFlags() const {
4973 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
4974 {{MONontemporalBit0, "riscv-nontemporal-domain-bit-0"},
4975 {MONontemporalBit1, "riscv-nontemporal-domain-bit-1"}};
4976 return ArrayRef(TargetFlags);
4977}
4978
4979unsigned RISCVInstrInfo::getTailDuplicateSize(CodeGenOptLevel OptLevel) const {
4980 return OptLevel >= CodeGenOptLevel::Aggressive
4981 ? STI.getTailDupAggressiveThreshold()
4982 : 2;
4983}
4984
4985bool RISCV::isRVVSpill(const MachineInstr &MI) {
4986 // RVV lacks any support for immediate addressing for stack addresses, so be
4987 // conservative.
4988 unsigned Opcode = MI.getOpcode();
4989 if (!RISCVVPseudosTable::getPseudoInfo(Pseudo: Opcode) &&
4990 !getLMULForRVVWholeLoadStore(Opcode) && !isRVVSpillForZvlsseg(Opcode))
4991 return false;
4992 return true;
4993}
4994
4995/// Return true if \p MI is a copy that will be lowered to one or more vmvNr.vs.
4996bool RISCV::isVectorCopy(const TargetRegisterInfo *TRI,
4997 const MachineInstr &MI) {
4998 return MI.isCopy() && MI.getOperand(i: 0).getReg().isPhysical() &&
4999 RISCVRegisterInfo::isRVVRegClass(
5000 RC: TRI->getMinimalPhysRegClass(Reg: MI.getOperand(i: 0).getReg()));
5001}
5002
5003std::optional<std::pair<unsigned, unsigned>>
5004RISCV::isRVVSpillForZvlsseg(unsigned Opcode) {
5005 switch (Opcode) {
5006 default:
5007 return std::nullopt;
5008 case RISCV::PseudoVSPILL2_M1:
5009 case RISCV::PseudoVRELOAD2_M1:
5010 return std::make_pair(x: 2u, y: 1u);
5011 case RISCV::PseudoVSPILL2_M2:
5012 case RISCV::PseudoVRELOAD2_M2:
5013 return std::make_pair(x: 2u, y: 2u);
5014 case RISCV::PseudoVSPILL2_M4:
5015 case RISCV::PseudoVRELOAD2_M4:
5016 return std::make_pair(x: 2u, y: 4u);
5017 case RISCV::PseudoVSPILL3_M1:
5018 case RISCV::PseudoVRELOAD3_M1:
5019 return std::make_pair(x: 3u, y: 1u);
5020 case RISCV::PseudoVSPILL3_M2:
5021 case RISCV::PseudoVRELOAD3_M2:
5022 return std::make_pair(x: 3u, y: 2u);
5023 case RISCV::PseudoVSPILL4_M1:
5024 case RISCV::PseudoVRELOAD4_M1:
5025 return std::make_pair(x: 4u, y: 1u);
5026 case RISCV::PseudoVSPILL4_M2:
5027 case RISCV::PseudoVRELOAD4_M2:
5028 return std::make_pair(x: 4u, y: 2u);
5029 case RISCV::PseudoVSPILL5_M1:
5030 case RISCV::PseudoVRELOAD5_M1:
5031 return std::make_pair(x: 5u, y: 1u);
5032 case RISCV::PseudoVSPILL6_M1:
5033 case RISCV::PseudoVRELOAD6_M1:
5034 return std::make_pair(x: 6u, y: 1u);
5035 case RISCV::PseudoVSPILL7_M1:
5036 case RISCV::PseudoVRELOAD7_M1:
5037 return std::make_pair(x: 7u, y: 1u);
5038 case RISCV::PseudoVSPILL8_M1:
5039 case RISCV::PseudoVRELOAD8_M1:
5040 return std::make_pair(x: 8u, y: 1u);
5041 }
5042}
5043
5044bool RISCV::hasEqualFRM(const MachineInstr &MI1, const MachineInstr &MI2) {
5045 int16_t MI1FrmOpIdx =
5046 RISCV::getNamedOperandIdx(Opcode: MI1.getOpcode(), Name: RISCV::OpName::frm);
5047 int16_t MI2FrmOpIdx =
5048 RISCV::getNamedOperandIdx(Opcode: MI2.getOpcode(), Name: RISCV::OpName::frm);
5049 if (MI1FrmOpIdx < 0 || MI2FrmOpIdx < 0)
5050 return false;
5051 MachineOperand FrmOp1 = MI1.getOperand(i: MI1FrmOpIdx);
5052 MachineOperand FrmOp2 = MI2.getOperand(i: MI2FrmOpIdx);
5053 return FrmOp1.getImm() == FrmOp2.getImm();
5054}
5055
5056std::optional<unsigned>
5057RISCV::getVectorLowDemandedScalarBits(unsigned Opcode, unsigned Log2SEW) {
5058 switch (Opcode) {
5059 default:
5060 return std::nullopt;
5061
5062 // 11.6. Vector Single-Width Shift Instructions
5063 case RISCV::VSLL_VX:
5064 case RISCV::VSRL_VX:
5065 case RISCV::VSRA_VX:
5066 // 12.4. Vector Single-Width Scaling Shift Instructions
5067 case RISCV::VSSRL_VX:
5068 case RISCV::VSSRA_VX:
5069 // Zvbb
5070 case RISCV::VROL_VX:
5071 case RISCV::VROR_VX:
5072 // Only the low lg2(SEW) bits of the shift-amount value are used.
5073 return Log2SEW;
5074
5075 // 11.7 Vector Narrowing Integer Right Shift Instructions
5076 case RISCV::VNSRL_WX:
5077 case RISCV::VNSRA_WX:
5078 // 12.5. Vector Narrowing Fixed-Point Clip Instructions
5079 case RISCV::VNCLIPU_WX:
5080 case RISCV::VNCLIP_WX:
5081 // Zvbb
5082 case RISCV::VWSLL_VX:
5083 // Only the low lg2(2*SEW) bits of the shift-amount value are used.
5084 return Log2SEW + 1;
5085
5086 // 11.1. Vector Single-Width Integer Add and Subtract
5087 case RISCV::VADD_VX:
5088 case RISCV::VSUB_VX:
5089 case RISCV::VRSUB_VX:
5090 // 11.2. Vector Widening Integer Add/Subtract
5091 case RISCV::VWADDU_VX:
5092 case RISCV::VWSUBU_VX:
5093 case RISCV::VWADD_VX:
5094 case RISCV::VWSUB_VX:
5095 case RISCV::VWADDU_WX:
5096 case RISCV::VWSUBU_WX:
5097 case RISCV::VWADD_WX:
5098 case RISCV::VWSUB_WX:
5099 // 11.4. Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions
5100 case RISCV::VADC_VXM:
5101 case RISCV::VADC_VIM:
5102 case RISCV::VMADC_VXM:
5103 case RISCV::VMADC_VIM:
5104 case RISCV::VMADC_VX:
5105 case RISCV::VSBC_VXM:
5106 case RISCV::VMSBC_VXM:
5107 case RISCV::VMSBC_VX:
5108 // 11.5 Vector Bitwise Logical Instructions
5109 case RISCV::VAND_VX:
5110 case RISCV::VOR_VX:
5111 case RISCV::VXOR_VX:
5112 // 11.8. Vector Integer Compare Instructions
5113 case RISCV::VMSEQ_VX:
5114 case RISCV::VMSNE_VX:
5115 case RISCV::VMSLTU_VX:
5116 case RISCV::VMSLT_VX:
5117 case RISCV::VMSLEU_VX:
5118 case RISCV::VMSLE_VX:
5119 case RISCV::VMSGTU_VX:
5120 case RISCV::VMSGT_VX:
5121 // 11.9. Vector Integer Min/Max Instructions
5122 case RISCV::VMINU_VX:
5123 case RISCV::VMIN_VX:
5124 case RISCV::VMAXU_VX:
5125 case RISCV::VMAX_VX:
5126 // 11.10. Vector Single-Width Integer Multiply Instructions
5127 case RISCV::VMUL_VX:
5128 case RISCV::VMULH_VX:
5129 case RISCV::VMULHU_VX:
5130 case RISCV::VMULHSU_VX:
5131 // 11.11. Vector Integer Divide Instructions
5132 case RISCV::VDIVU_VX:
5133 case RISCV::VDIV_VX:
5134 case RISCV::VREMU_VX:
5135 case RISCV::VREM_VX:
5136 // 11.12. Vector Widening Integer Multiply Instructions
5137 case RISCV::VWMUL_VX:
5138 case RISCV::VWMULU_VX:
5139 case RISCV::VWMULSU_VX:
5140 // 11.13. Vector Single-Width Integer Multiply-Add Instructions
5141 case RISCV::VMACC_VX:
5142 case RISCV::VNMSAC_VX:
5143 case RISCV::VMADD_VX:
5144 case RISCV::VNMSUB_VX:
5145 // 11.14. Vector Widening Integer Multiply-Add Instructions
5146 case RISCV::VWMACCU_VX:
5147 case RISCV::VWMACC_VX:
5148 case RISCV::VWMACCSU_VX:
5149 case RISCV::VWMACCUS_VX:
5150 // 11.15. Vector Integer Merge Instructions
5151 case RISCV::VMERGE_VXM:
5152 // 11.16. Vector Integer Move Instructions
5153 case RISCV::VMV_V_X:
5154 // 12.1. Vector Single-Width Saturating Add and Subtract
5155 case RISCV::VSADDU_VX:
5156 case RISCV::VSADD_VX:
5157 case RISCV::VSSUBU_VX:
5158 case RISCV::VSSUB_VX:
5159 // 12.2. Vector Single-Width Averaging Add and Subtract
5160 case RISCV::VAADDU_VX:
5161 case RISCV::VAADD_VX:
5162 case RISCV::VASUBU_VX:
5163 case RISCV::VASUB_VX:
5164 // 12.3. Vector Single-Width Fractional Multiply with Rounding and Saturation
5165 case RISCV::VSMUL_VX:
5166 // 16.1. Integer Scalar Move Instructions
5167 case RISCV::VMV_S_X:
5168 // Zvbb
5169 case RISCV::VANDN_VX:
5170 return 1U << Log2SEW;
5171 }
5172}
5173
5174unsigned RISCV::getRVVMCOpcode(unsigned RVVPseudoOpcode) {
5175 const RISCVVPseudosTable::PseudoInfo *RVV =
5176 RISCVVPseudosTable::getPseudoInfo(Pseudo: RVVPseudoOpcode);
5177 if (!RVV)
5178 return 0;
5179 return RVV->BaseInstr;
5180}
5181
5182unsigned RISCV::getDestLog2EEW(const MCInstrDesc &Desc, unsigned Log2SEW) {
5183 unsigned DestEEW =
5184 (Desc.TSFlags & RISCVII::DestEEWMask) >> RISCVII::DestEEWShift;
5185 // EEW = 1
5186 if (DestEEW == 0)
5187 return 0;
5188 // EEW = SEW * n
5189 unsigned Scaled = Log2SEW + (DestEEW - 1);
5190 assert(Scaled >= 3 && Scaled <= 6);
5191 return Scaled;
5192}
5193
5194static std::optional<int64_t> getEffectiveImm(const MachineOperand &MO) {
5195 assert(MO.isImm() || MO.getReg().isVirtual());
5196 if (MO.isImm())
5197 return MO.getImm();
5198 const MachineInstr *Def =
5199 MO.getParent()->getMF()->getRegInfo().getVRegDef(Reg: MO.getReg());
5200 int64_t Imm;
5201 if (isLoadImm(MI: Def, Imm))
5202 return Imm;
5203 return std::nullopt;
5204}
5205
5206/// Given two VL operands, do we know that LHS <= RHS? Must be used in SSA form.
5207bool RISCV::isVLKnownLE(const MachineOperand &LHS, const MachineOperand &RHS) {
5208 assert((LHS.isImm() || LHS.getParent()->getMF()->getRegInfo().isSSA()) &&
5209 (RHS.isImm() || RHS.getParent()->getMF()->getRegInfo().isSSA()));
5210 if (LHS.isReg() && RHS.isReg() && LHS.getReg().isVirtual() &&
5211 LHS.getReg() == RHS.getReg())
5212 return true;
5213 if (RHS.isImm() && RHS.getImm() == RISCV::VLMaxSentinel)
5214 return true;
5215 if (LHS.isImm() && LHS.getImm() == 0)
5216 return true;
5217 if (LHS.isImm() && LHS.getImm() == RISCV::VLMaxSentinel)
5218 return false;
5219 std::optional<int64_t> LHSImm = getEffectiveImm(MO: LHS),
5220 RHSImm = getEffectiveImm(MO: RHS);
5221 if (!LHSImm || !RHSImm)
5222 return false;
5223 return LHSImm <= RHSImm;
5224}
5225
5226namespace {
5227class RISCVPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
5228 const MachineInstr *LHS;
5229 const MachineInstr *RHS;
5230 SmallVector<MachineOperand, 3> Cond;
5231
5232public:
5233 RISCVPipelinerLoopInfo(const MachineInstr *LHS, const MachineInstr *RHS,
5234 const SmallVectorImpl<MachineOperand> &Cond)
5235 : LHS(LHS), RHS(RHS), Cond(Cond.begin(), Cond.end()) {}
5236
5237 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
5238 // Make the instructions for loop control be placed in stage 0.
5239 // The predecessors of LHS/RHS are considered by the caller.
5240 if (LHS && MI == LHS)
5241 return true;
5242 if (RHS && MI == RHS)
5243 return true;
5244 return false;
5245 }
5246
5247 std::optional<bool> createTripCountGreaterCondition(
5248 int TC, MachineBasicBlock &MBB,
5249 SmallVectorImpl<MachineOperand> &CondParam) override {
5250 // A branch instruction will be inserted as "if (Cond) goto epilogue".
5251 // Cond is normalized for such use.
5252 // The predecessors of the branch are assumed to have already been inserted.
5253 CondParam = Cond;
5254 return {};
5255 }
5256
5257 void setPreheader(MachineBasicBlock *NewPreheader) override {}
5258
5259 void adjustTripCount(int TripCountAdjust) override {}
5260};
5261} // namespace
5262
5263std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
5264RISCVInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
5265 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
5266 SmallVector<MachineOperand, 4> Cond;
5267 if (analyzeBranch(MBB&: *LoopBB, TBB, FBB, Cond, /*AllowModify=*/false))
5268 return nullptr;
5269
5270 // Infinite loops are not supported
5271 if (TBB == LoopBB && FBB == LoopBB)
5272 return nullptr;
5273
5274 // Must be conditional branch
5275 if (FBB == nullptr)
5276 return nullptr;
5277
5278 assert((TBB == LoopBB || FBB == LoopBB) &&
5279 "The Loop must be a single-basic-block loop");
5280
5281 // Normalization for createTripCountGreaterCondition()
5282 if (TBB == LoopBB)
5283 reverseBranchCondition(Cond);
5284
5285 const MachineRegisterInfo &MRI = LoopBB->getParent()->getRegInfo();
5286 auto FindRegDef = [&MRI](MachineOperand &Op) -> const MachineInstr * {
5287 if (!Op.isReg())
5288 return nullptr;
5289 Register Reg = Op.getReg();
5290 if (!Reg.isVirtual())
5291 return nullptr;
5292 return MRI.getVRegDef(Reg);
5293 };
5294
5295 const MachineInstr *LHS = FindRegDef(Cond[1]);
5296 const MachineInstr *RHS = FindRegDef(Cond[2]);
5297 if (LHS && LHS->isPHI())
5298 return nullptr;
5299 if (RHS && RHS->isPHI())
5300 return nullptr;
5301
5302 return std::make_unique<RISCVPipelinerLoopInfo>(args&: LHS, args&: RHS, args&: Cond);
5303}
5304
5305// FIXME: We should remove this if we have a default generic scheduling model.
5306bool RISCVInstrInfo::isHighLatencyDef(int Opc) const {
5307 unsigned RVVMCOpcode = RISCV::getRVVMCOpcode(RVVPseudoOpcode: Opc);
5308 Opc = RVVMCOpcode ? RVVMCOpcode : Opc;
5309 switch (Opc) {
5310 default:
5311 return false;
5312 // Integer div/rem.
5313 case RISCV::DIV:
5314 case RISCV::DIVW:
5315 case RISCV::DIVU:
5316 case RISCV::DIVUW:
5317 case RISCV::REM:
5318 case RISCV::REMW:
5319 case RISCV::REMU:
5320 case RISCV::REMUW:
5321 // Floating-point div/sqrt.
5322 case RISCV::FDIV_H:
5323 case RISCV::FDIV_S:
5324 case RISCV::FDIV_D:
5325 case RISCV::FDIV_H_INX:
5326 case RISCV::FDIV_S_INX:
5327 case RISCV::FDIV_D_INX:
5328 case RISCV::FDIV_D_IN32X:
5329 case RISCV::FSQRT_H:
5330 case RISCV::FSQRT_S:
5331 case RISCV::FSQRT_D:
5332 case RISCV::FSQRT_H_INX:
5333 case RISCV::FSQRT_S_INX:
5334 case RISCV::FSQRT_D_INX:
5335 case RISCV::FSQRT_D_IN32X:
5336 // Vector integer div/rem
5337 case RISCV::VDIV_VV:
5338 case RISCV::VDIV_VX:
5339 case RISCV::VDIVU_VV:
5340 case RISCV::VDIVU_VX:
5341 case RISCV::VREM_VV:
5342 case RISCV::VREM_VX:
5343 case RISCV::VREMU_VV:
5344 case RISCV::VREMU_VX:
5345 // Vector floating-point div/sqrt.
5346 case RISCV::VFDIV_VV:
5347 case RISCV::VFDIV_VF:
5348 case RISCV::VFRDIV_VF:
5349 case RISCV::VFSQRT_V:
5350 case RISCV::VFRSQRT7_V:
5351 return true;
5352 }
5353}
5354
5355bool RISCVInstrInfo::isVRegCopy(const MachineInstr *MI, unsigned LMul) const {
5356 if (MI->getOpcode() != TargetOpcode::COPY)
5357 return false;
5358 const MachineRegisterInfo &MRI = MI->getMF()->getRegInfo();
5359 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
5360
5361 Register DstReg = MI->getOperand(i: 0).getReg();
5362 const TargetRegisterClass *RC = DstReg.isVirtual()
5363 ? MRI.getRegClass(Reg: DstReg)
5364 : TRI->getMinimalPhysRegClass(Reg: DstReg);
5365
5366 if (!RISCVRegisterInfo::isRVVRegClass(RC))
5367 return false;
5368
5369 if (!LMul)
5370 return true;
5371
5372 // TODO: Perhaps we could distinguish segment register classes (e.g. VRN3M2)
5373 // in the future.
5374 auto [RCLMul, RCFractional] =
5375 RISCVVType::decodeVLMUL(VLMul: RISCVRI::getLMul(TSFlags: RC->TSFlags));
5376 return (!RCFractional && LMul == RCLMul) || (RCFractional && LMul == 1);
5377}
5378
5379bool RISCVInstrInfo::requiresNTLHint(const MachineInstr &MI) const {
5380 if (MI.memoperands_empty())
5381 return false;
5382
5383 MachineMemOperand *MMO = *(MI.memoperands_begin());
5384 if (!MMO->isNonTemporal())
5385 return false;
5386
5387 return true;
5388}
5389
5390bool RISCVInstrInfo::isSafeToMove(const MachineInstr &From,
5391 const MachineInstr &To) {
5392 assert(From.getParent() == To.getParent());
5393 SmallVector<Register> PhysUses, PhysDefs;
5394 for (const MachineOperand &MO : From.all_uses())
5395 if (MO.getReg().isPhysical())
5396 PhysUses.push_back(Elt: MO.getReg());
5397 for (const MachineOperand &MO : From.all_defs())
5398 if (MO.getReg().isPhysical())
5399 PhysDefs.push_back(Elt: MO.getReg());
5400 bool SawStore = false;
5401 for (auto II = std::next(x: From.getIterator()); II != To.getIterator(); II++) {
5402 for (Register PhysReg : PhysUses)
5403 if (II->definesRegister(Reg: PhysReg, TRI: nullptr))
5404 return false;
5405 for (Register PhysReg : PhysDefs)
5406 if (II->definesRegister(Reg: PhysReg, TRI: nullptr) ||
5407 II->readsRegister(Reg: PhysReg, TRI: nullptr))
5408 return false;
5409 if (II->mayStore()) {
5410 SawStore = true;
5411 break;
5412 }
5413 }
5414 return From.isSafeToMove(SawStore);
5415}
5416