1//===-- RISCVInstrInfo.cpp - RISC-V Instruction Information -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the RISC-V implementation of the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RISCVInstrInfo.h"
14#include "MCTargetDesc/RISCVBaseInfo.h"
15#include "MCTargetDesc/RISCVMatInt.h"
16#include "RISCV.h"
17#include "RISCVMachineFunctionInfo.h"
18#include "RISCVSubtarget.h"
19#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/ADT/Statistic.h"
22#include "llvm/Analysis/MemoryLocation.h"
23#include "llvm/Analysis/ValueTracking.h"
24#include "llvm/CodeGen/LiveIntervals.h"
25#include "llvm/CodeGen/MachineCombinerPattern.h"
26#include "llvm/CodeGen/MachineInstrBuilder.h"
27#include "llvm/CodeGen/MachineRegisterInfo.h"
28#include "llvm/CodeGen/MachineTraceMetrics.h"
29#include "llvm/CodeGen/RegisterScavenging.h"
30#include "llvm/CodeGen/StackMaps.h"
31#include "llvm/IR/DebugInfoMetadata.h"
32#include "llvm/IR/Module.h"
33#include "llvm/MC/MCDwarf.h"
34#include "llvm/MC/MCInstBuilder.h"
35#include "llvm/MC/TargetRegistry.h"
36#include "llvm/Support/ErrorHandling.h"
37
38using namespace llvm;
39
40#define GEN_CHECK_COMPRESS_INSTR
41#include "RISCVGenCompressInstEmitter.inc"
42
43#define GET_INSTRINFO_CTOR_DTOR
44#include "RISCVGenInstrInfo.inc"
45
46#define DEBUG_TYPE "riscv-instr-info"
47STATISTIC(NumVRegSpilled,
48 "Number of registers within vector register groups spilled");
49STATISTIC(NumVRegReloaded,
50 "Number of registers within vector register groups reloaded");
51
52namespace llvm::RISCVVPseudosTable {
53
54using namespace RISCV;
55
56#define GET_RISCVVPseudosTable_IMPL
57#include "RISCVGenSearchableTables.inc"
58
59} // namespace llvm::RISCVVPseudosTable
60
61namespace llvm::RISCV {
62
63#define GET_RISCVMaskedPseudosTable_IMPL
64#include "RISCVGenSearchableTables.inc"
65
66} // end namespace llvm::RISCV
67
68RISCVInstrInfo::RISCVInstrInfo(const RISCVSubtarget &STI)
69 : RISCVGenInstrInfo(STI, RegInfo, RISCV::ADJCALLSTACKDOWN,
70 RISCV::ADJCALLSTACKUP),
71 RegInfo(STI.getHwMode(), STI.getUserReservedXRegs()), STI(STI) {}
72
73#define GET_INSTRINFO_HELPERS
74#include "RISCVGenInstrInfo.inc"
75
76MCInst RISCVInstrInfo::getNop() const {
77 if (STI.hasStdExtZca())
78 return MCInstBuilder(RISCV::C_NOP);
79 return MCInstBuilder(RISCV::ADDI)
80 .addReg(Reg: RISCV::X0)
81 .addReg(Reg: RISCV::X0)
82 .addImm(Val: 0);
83}
84
85Register RISCVInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
86 int &FrameIndex) const {
87 TypeSize Dummy = TypeSize::getZero();
88 return isLoadFromStackSlot(MI, FrameIndex, MemBytes&: Dummy);
89}
90
91static std::optional<unsigned> getLMULForRVVWholeLoadStore(unsigned Opcode) {
92 switch (Opcode) {
93 default:
94 return std::nullopt;
95 case RISCV::VS1R_V:
96 case RISCV::VL1RE8_V:
97 case RISCV::VL1RE16_V:
98 case RISCV::VL1RE32_V:
99 case RISCV::VL1RE64_V:
100 return 1;
101 case RISCV::VS2R_V:
102 case RISCV::VL2RE8_V:
103 case RISCV::VL2RE16_V:
104 case RISCV::VL2RE32_V:
105 case RISCV::VL2RE64_V:
106 return 2;
107 case RISCV::VS4R_V:
108 case RISCV::VL4RE8_V:
109 case RISCV::VL4RE16_V:
110 case RISCV::VL4RE32_V:
111 case RISCV::VL4RE64_V:
112 return 4;
113 case RISCV::VS8R_V:
114 case RISCV::VL8RE8_V:
115 case RISCV::VL8RE16_V:
116 case RISCV::VL8RE32_V:
117 case RISCV::VL8RE64_V:
118 return 8;
119 }
120}
121
122Register RISCVInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
123 int &FrameIndex,
124 TypeSize &MemBytes) const {
125 switch (MI.getOpcode()) {
126 default:
127 return 0;
128 case RISCV::LB:
129 case RISCV::LBU:
130 MemBytes = TypeSize::getFixed(ExactSize: 1);
131 break;
132 case RISCV::LH:
133 case RISCV::LH_INX:
134 case RISCV::LHU:
135 case RISCV::FLH:
136 MemBytes = TypeSize::getFixed(ExactSize: 2);
137 break;
138 case RISCV::LW:
139 case RISCV::LW_INX:
140 case RISCV::FLW:
141 case RISCV::LWU:
142 MemBytes = TypeSize::getFixed(ExactSize: 4);
143 break;
144 case RISCV::LD:
145 case RISCV::LD_RV32:
146 case RISCV::FLD:
147 MemBytes = TypeSize::getFixed(ExactSize: 8);
148 break;
149 case RISCV::VL1RE8_V:
150 case RISCV::VL2RE8_V:
151 case RISCV::VL4RE8_V:
152 case RISCV::VL8RE8_V:
153 if (!MI.getOperand(i: 1).isFI())
154 return Register();
155 FrameIndex = MI.getOperand(i: 1).getIndex();
156 unsigned LMUL = *getLMULForRVVWholeLoadStore(Opcode: MI.getOpcode());
157 MemBytes = TypeSize::getScalable(MinimumSize: RISCV::RVVBytesPerBlock * LMUL);
158 return MI.getOperand(i: 0).getReg();
159 }
160
161 if (MI.getOperand(i: 1).isFI() && MI.getOperand(i: 2).isImm() &&
162 MI.getOperand(i: 2).getImm() == 0) {
163 FrameIndex = MI.getOperand(i: 1).getIndex();
164 return MI.getOperand(i: 0).getReg();
165 }
166
167 return 0;
168}
169
170Register RISCVInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
171 int &FrameIndex) const {
172 TypeSize Dummy = TypeSize::getZero();
173 return isStoreToStackSlot(MI, FrameIndex, MemBytes&: Dummy);
174}
175
176Register RISCVInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
177 int &FrameIndex,
178 TypeSize &MemBytes) const {
179 switch (MI.getOpcode()) {
180 default:
181 return 0;
182 case RISCV::SB:
183 MemBytes = TypeSize::getFixed(ExactSize: 1);
184 break;
185 case RISCV::SH:
186 case RISCV::SH_INX:
187 case RISCV::FSH:
188 MemBytes = TypeSize::getFixed(ExactSize: 2);
189 break;
190 case RISCV::SW:
191 case RISCV::SW_INX:
192 case RISCV::FSW:
193 MemBytes = TypeSize::getFixed(ExactSize: 4);
194 break;
195 case RISCV::SD:
196 case RISCV::SD_RV32:
197 case RISCV::FSD:
198 MemBytes = TypeSize::getFixed(ExactSize: 8);
199 break;
200 case RISCV::VS1R_V:
201 case RISCV::VS2R_V:
202 case RISCV::VS4R_V:
203 case RISCV::VS8R_V:
204 if (!MI.getOperand(i: 1).isFI())
205 return Register();
206 FrameIndex = MI.getOperand(i: 1).getIndex();
207 unsigned LMUL = *getLMULForRVVWholeLoadStore(Opcode: MI.getOpcode());
208 MemBytes = TypeSize::getScalable(MinimumSize: RISCV::RVVBytesPerBlock * LMUL);
209 return MI.getOperand(i: 0).getReg();
210 }
211
212 if (MI.getOperand(i: 1).isFI() && MI.getOperand(i: 2).isImm() &&
213 MI.getOperand(i: 2).getImm() == 0) {
214 FrameIndex = MI.getOperand(i: 1).getIndex();
215 return MI.getOperand(i: 0).getReg();
216 }
217
218 return 0;
219}
220
221bool RISCVInstrInfo::isReMaterializableImpl(
222 const MachineInstr &MI) const {
223 switch (RISCV::getRVVMCOpcode(RVVPseudoOpcode: MI.getOpcode())) {
224 case RISCV::VMV_V_X:
225 case RISCV::VFMV_V_F:
226 case RISCV::VMV_V_I:
227 case RISCV::VMV_S_X:
228 case RISCV::VFMV_S_F:
229 case RISCV::VID_V:
230 return MI.getOperand(i: 1).isUndef();
231 default:
232 return TargetInstrInfo::isReMaterializableImpl(MI);
233 }
234}
235
236static bool forwardCopyWillClobberTuple(unsigned DstReg, unsigned SrcReg,
237 unsigned NumRegs) {
238 return DstReg > SrcReg && (DstReg - SrcReg) < NumRegs;
239}
240
241static bool isConvertibleToVMV_V_V(const RISCVSubtarget &STI,
242 const MachineBasicBlock &MBB,
243 MachineBasicBlock::const_iterator MBBI,
244 MachineBasicBlock::const_iterator &DefMBBI,
245 RISCVVType::VLMUL LMul) {
246 if (STI.getCLOpts().prefer_whole_register_move)
247 return false;
248
249 assert(MBBI->getOpcode() == TargetOpcode::COPY &&
250 "Unexpected COPY instruction.");
251 Register SrcReg = MBBI->getOperand(i: 1).getReg();
252 const TargetRegisterInfo *TRI = STI.getRegisterInfo();
253
254 bool FoundDef = false;
255 bool FirstVSetVLI = false;
256 unsigned FirstSEW = 0;
257 while (MBBI != MBB.begin()) {
258 --MBBI;
259 if (MBBI->isMetaInstruction())
260 continue;
261
262 if (RISCVInstrInfo::isVectorConfigInstr(MI: *MBBI)) {
263 // There is a vsetvli between COPY and source define instruction.
264 // vy = def_vop ... (producing instruction)
265 // ...
266 // vsetvli
267 // ...
268 // vx = COPY vy
269 if (!FoundDef) {
270 if (!FirstVSetVLI) {
271 FirstVSetVLI = true;
272 unsigned FirstVType = MBBI->getOperand(i: 2).getImm();
273 RISCVVType::VLMUL FirstLMul = RISCVVType::getVLMUL(VType: FirstVType);
274 FirstSEW = RISCVVType::getSEW(VType: FirstVType);
275 // The first encountered vsetvli must have the same lmul as the
276 // register class of COPY.
277 if (FirstLMul != LMul)
278 return false;
279 }
280 // Only permit `vsetvli x0, x0, vtype` between COPY and the source
281 // define instruction.
282 if (!RISCVInstrInfo::isVLPreservingConfig(MI: *MBBI))
283 return false;
284 continue;
285 }
286
287 // MBBI is the first vsetvli before the producing instruction.
288 unsigned VType = MBBI->getOperand(i: 2).getImm();
289 // If there is a vsetvli between COPY and the producing instruction.
290 if (FirstVSetVLI) {
291 // If SEW is different, return false.
292 if (RISCVVType::getSEW(VType) != FirstSEW)
293 return false;
294 }
295
296 // If the vsetvli is tail undisturbed, keep the whole register move.
297 if (!RISCVVType::isTailAgnostic(VType))
298 return false;
299
300 // The checking is conservative. We only have register classes for
301 // LMUL = 1/2/4/8. We should be able to convert vmv1r.v to vmv.v.v
302 // for fractional LMUL operations. However, we could not use the vsetvli
303 // lmul for widening operations. The result of widening operation is
304 // 2 x LMUL.
305 return LMul == RISCVVType::getVLMUL(VType);
306 } else if (MBBI->isInlineAsm() || MBBI->isCall()) {
307 return false;
308 } else if (MBBI->getNumDefs()) {
309 // Check all the instructions which will change VL.
310 // For example, vleff has implicit def VL.
311 if (MBBI->modifiesRegister(Reg: RISCV::VL, /*TRI=*/nullptr))
312 return false;
313
314 // Only converting whole register copies to vmv.v.v when the defining
315 // value appears in the explicit operands.
316 for (const MachineOperand &MO : MBBI->explicit_operands()) {
317 if (!MO.isReg() || !MO.isDef())
318 continue;
319 if (!FoundDef && TRI->regsOverlap(RegA: MO.getReg(), RegB: SrcReg)) {
320 // We only permit the source of COPY has the same LMUL as the defined
321 // operand.
322 // There are cases we need to keep the whole register copy if the LMUL
323 // is different.
324 // For example,
325 // $x0 = PseudoVSETIVLI 4, 73 // vsetivli zero, 4, e16,m2,ta,m
326 // $v28m4 = PseudoVWADD_VV_M2 $v26m2, $v8m2
327 // # The COPY may be created by vlmul_trunc intrinsic.
328 // $v26m2 = COPY renamable $v28m2, implicit killed $v28m4
329 //
330 // After widening, the valid value will be 4 x e32 elements. If we
331 // convert the COPY to vmv.v.v, it will only copy 4 x e16 elements.
332 // FIXME: The COPY of subregister of Zvlsseg register will not be able
333 // to convert to vmv.v.[v|i] under the constraint.
334 if (MO.getReg() != SrcReg)
335 return false;
336
337 // In widening reduction instructions with LMUL_1 input vector case,
338 // only checking the LMUL is insufficient due to reduction result is
339 // always LMUL_1.
340 // For example,
341 // $x11 = PseudoVSETIVLI 1, 64 // vsetivli a1, 1, e8, m1, ta, mu
342 // $v8m1 = PseudoVWREDSUM_VS_M1 $v26, $v27
343 // $v26 = COPY killed renamable $v8
344 // After widening, The valid value will be 1 x e16 elements. If we
345 // convert the COPY to vmv.v.v, it will only copy 1 x e8 elements.
346 uint64_t TSFlags = MBBI->getDesc().TSFlags;
347 if (RISCVII::isRVVWideningReduction(TSFlags))
348 return false;
349
350 // If the producing instruction does not depend on vsetvli, do not
351 // convert COPY to vmv.v.v. For example, VL1R_V or PseudoVRELOAD.
352 if (!RISCVII::hasSEWOp(TSFlags) || !RISCVII::hasVLOp(TSFlags))
353 return false;
354
355 // Found the definition.
356 FoundDef = true;
357 DefMBBI = MBBI;
358 break;
359 }
360 }
361 }
362 }
363
364 return false;
365}
366
367void RISCVInstrInfo::copyPhysRegVector(
368 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
369 const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc,
370 const TargetRegisterClass *RegClass) const {
371 const RISCVRegisterInfo *TRI = STI.getRegisterInfo();
372 RISCVVType::VLMUL LMul = RISCVRI::getLMul(TSFlags: RegClass->TSFlags);
373 unsigned NF = RISCVRI::getNF(TSFlags: RegClass->TSFlags);
374
375 uint16_t SrcEncoding = TRI->getEncodingValue(Reg: SrcReg);
376 uint16_t DstEncoding = TRI->getEncodingValue(Reg: DstReg);
377 auto [LMulVal, Fractional] = RISCVVType::decodeVLMUL(VLMul: LMul);
378 assert(!Fractional && "It is impossible be fractional lmul here.");
379 unsigned NumRegs = NF * LMulVal;
380 bool ReversedCopy =
381 forwardCopyWillClobberTuple(DstReg: DstEncoding, SrcReg: SrcEncoding, NumRegs);
382 if (ReversedCopy) {
383 // If the src and dest overlap when copying a tuple, we need to copy the
384 // registers in reverse.
385 SrcEncoding += NumRegs - 1;
386 DstEncoding += NumRegs - 1;
387 }
388
389 unsigned I = 0;
390 auto GetCopyInfo = [&](uint16_t SrcEncoding, uint16_t DstEncoding)
391 -> std::tuple<RISCVVType::VLMUL, const TargetRegisterClass &, unsigned,
392 unsigned, unsigned> {
393 if (ReversedCopy) {
394 // For reversed copying, if there are enough aligned registers(8/4/2), we
395 // can do a larger copy(LMUL8/4/2).
396 // Besides, we have already known that DstEncoding is larger than
397 // SrcEncoding in forwardCopyWillClobberTuple, so the difference between
398 // DstEncoding and SrcEncoding should be >= LMUL value we try to use to
399 // avoid clobbering.
400 uint16_t Diff = DstEncoding - SrcEncoding;
401 if (I + 8 <= NumRegs && Diff >= 8 && SrcEncoding % 8 == 7 &&
402 DstEncoding % 8 == 7)
403 return {RISCVVType::LMUL_8, RISCV::VRM8RegClass, RISCV::VMV8R_V,
404 RISCV::PseudoVMV_V_V_M8, RISCV::PseudoVMV_V_I_M8};
405 if (I + 4 <= NumRegs && Diff >= 4 && SrcEncoding % 4 == 3 &&
406 DstEncoding % 4 == 3)
407 return {RISCVVType::LMUL_4, RISCV::VRM4RegClass, RISCV::VMV4R_V,
408 RISCV::PseudoVMV_V_V_M4, RISCV::PseudoVMV_V_I_M4};
409 if (I + 2 <= NumRegs && Diff >= 2 && SrcEncoding % 2 == 1 &&
410 DstEncoding % 2 == 1)
411 return {RISCVVType::LMUL_2, RISCV::VRM2RegClass, RISCV::VMV2R_V,
412 RISCV::PseudoVMV_V_V_M2, RISCV::PseudoVMV_V_I_M2};
413 // Or we should do LMUL1 copying.
414 return {RISCVVType::LMUL_1, RISCV::VRRegClass, RISCV::VMV1R_V,
415 RISCV::PseudoVMV_V_V_M1, RISCV::PseudoVMV_V_I_M1};
416 }
417
418 // For forward copying, if source register encoding and destination register
419 // encoding are aligned to 8/4/2, we can do a LMUL8/4/2 copying.
420 if (I + 8 <= NumRegs && SrcEncoding % 8 == 0 && DstEncoding % 8 == 0)
421 return {RISCVVType::LMUL_8, RISCV::VRM8RegClass, RISCV::VMV8R_V,
422 RISCV::PseudoVMV_V_V_M8, RISCV::PseudoVMV_V_I_M8};
423 if (I + 4 <= NumRegs && SrcEncoding % 4 == 0 && DstEncoding % 4 == 0)
424 return {RISCVVType::LMUL_4, RISCV::VRM4RegClass, RISCV::VMV4R_V,
425 RISCV::PseudoVMV_V_V_M4, RISCV::PseudoVMV_V_I_M4};
426 if (I + 2 <= NumRegs && SrcEncoding % 2 == 0 && DstEncoding % 2 == 0)
427 return {RISCVVType::LMUL_2, RISCV::VRM2RegClass, RISCV::VMV2R_V,
428 RISCV::PseudoVMV_V_V_M2, RISCV::PseudoVMV_V_I_M2};
429 // Or we should do LMUL1 copying.
430 return {RISCVVType::LMUL_1, RISCV::VRRegClass, RISCV::VMV1R_V,
431 RISCV::PseudoVMV_V_V_M1, RISCV::PseudoVMV_V_I_M1};
432 };
433
434 while (I != NumRegs) {
435 // For non-segment copying, we only do this once as the registers are always
436 // aligned.
437 // For segment copying, we may do this several times. If the registers are
438 // aligned to larger LMUL, we can eliminate some copyings.
439 auto [LMulCopied, RegClass, Opc, VVOpc, VIOpc] =
440 GetCopyInfo(SrcEncoding, DstEncoding);
441 auto [NumCopied, _] = RISCVVType::decodeVLMUL(VLMul: LMulCopied);
442
443 MachineBasicBlock::const_iterator DefMBBI;
444 if (LMul == LMulCopied &&
445 isConvertibleToVMV_V_V(STI, MBB, MBBI, DefMBBI, LMul)) {
446 Opc = VVOpc;
447 if (DefMBBI->getOpcode() == VIOpc)
448 Opc = VIOpc;
449 }
450
451 // Emit actual copying.
452 // For reversed copying, the encoding should be decreased.
453 MCRegister ActualSrcReg = TRI->findVRegWithEncoding(
454 RegClass, Encoding: ReversedCopy ? (SrcEncoding - NumCopied + 1) : SrcEncoding);
455 MCRegister ActualDstReg = TRI->findVRegWithEncoding(
456 RegClass, Encoding: ReversedCopy ? (DstEncoding - NumCopied + 1) : DstEncoding);
457
458 auto MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: Opc), DestReg: ActualDstReg);
459 bool UseVMV_V_I = RISCV::getRVVMCOpcode(RVVPseudoOpcode: Opc) == RISCV::VMV_V_I;
460 bool UseVMV = UseVMV_V_I || RISCV::getRVVMCOpcode(RVVPseudoOpcode: Opc) == RISCV::VMV_V_V;
461 if (UseVMV)
462 MIB.addReg(RegNo: ActualDstReg, Flags: RegState::Undef);
463 if (UseVMV_V_I)
464 MIB = MIB.add(MO: DefMBBI->getOperand(i: 2));
465 else
466 MIB = MIB.addReg(RegNo: ActualSrcReg, Flags: getKillRegState(B: KillSrc));
467 if (UseVMV) {
468 const MCInstrDesc &Desc = DefMBBI->getDesc();
469 MIB.add(MO: DefMBBI->getOperand(i: RISCVII::getVLOpNum(Desc))); // AVL
470 unsigned Log2SEW =
471 DefMBBI->getOperand(i: RISCVII::getSEWOpNum(Desc)).getImm();
472 MIB.addImm(Val: Log2SEW ? Log2SEW : 3); // SEW
473 MIB.addImm(Val: 0); // tu, mu
474 MIB.addReg(RegNo: RISCV::VL, Flags: RegState::Implicit);
475 MIB.addReg(RegNo: RISCV::VTYPE, Flags: RegState::Implicit);
476 }
477 // Add an implicit read of the original source to silence the verifier
478 // in the cases where some of the smaller VRs we're copying from might be
479 // undef, caused by the fact that the original, larger source VR might not
480 // be fully initialized at the time this COPY happens.
481 MIB.addReg(RegNo: SrcReg, Flags: RegState::Implicit);
482
483 // If we are copying reversely, we should decrease the encoding.
484 SrcEncoding += (ReversedCopy ? -NumCopied : NumCopied);
485 DstEncoding += (ReversedCopy ? -NumCopied : NumCopied);
486 I += NumCopied;
487 }
488}
489
490void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
491 MachineBasicBlock::iterator MBBI,
492 const DebugLoc &DL, Register DstReg,
493 Register SrcReg, bool KillSrc,
494 bool RenamableDest, bool RenamableSrc) const {
495 const RISCVRegisterInfo *TRI = STI.getRegisterInfo();
496 RegState KillFlag = getKillRegState(B: KillSrc);
497
498 if (RISCV::GPRRegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
499 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::ADDI), DestReg: DstReg)
500 .addReg(RegNo: SrcReg, Flags: KillFlag | getRenamableRegState(B: RenamableSrc))
501 .addImm(Val: 0);
502 return;
503 }
504
505 if (RISCV::GPRF16RegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
506 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::PseudoMV_FPR16INX), DestReg: DstReg)
507 .addReg(RegNo: SrcReg, Flags: KillFlag | getRenamableRegState(B: RenamableSrc));
508 return;
509 }
510
511 if (RISCV::GPRF32RegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
512 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::PseudoMV_FPR32INX), DestReg: DstReg)
513 .addReg(RegNo: SrcReg, Flags: KillFlag | getRenamableRegState(B: RenamableSrc));
514 return;
515 }
516
517 if (RISCV::GPRPairRegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
518 if (!STI.is64Bit()) {
519 if (STI.hasStdExtZdinx()) {
520 // On RV32_Zdinx, FMV.D will move a pair of registers to another pair of
521 // registers, in one instruction.
522 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FSGNJ_D_IN32X), DestReg: DstReg)
523 .addReg(RegNo: SrcReg, Flags: getRenamableRegState(B: RenamableSrc))
524 .addReg(RegNo: SrcReg, Flags: KillFlag | getRenamableRegState(B: RenamableSrc));
525 return;
526 }
527
528 if (STI.hasStdExtP()) {
529 // On RV32P, `padd.dw` is a GPR Pair Add
530 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::PADD_DW), DestReg: DstReg)
531 .addReg(RegNo: RISCV::X0_Pair)
532 .addReg(RegNo: SrcReg, Flags: KillFlag | getRenamableRegState(B: RenamableSrc));
533 return;
534 }
535 }
536
537 auto [EvenReg, OddReg] = TRI->getGPRPairSubRegs(Pair: SrcReg);
538 assert(DstReg != RISCV::X0_Pair && "Cannot write to X0_Pair");
539
540 // Emit an ADDI for both parts of GPRPair.
541 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::ADDI),
542 DestReg: TRI->getSubReg(Reg: DstReg, Idx: RISCV::sub_gpr_even))
543 .addReg(RegNo: EvenReg, Flags: KillFlag)
544 .addImm(Val: 0);
545 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::ADDI),
546 DestReg: TRI->getSubReg(Reg: DstReg, Idx: RISCV::sub_gpr_odd))
547 .addReg(RegNo: OddReg, Flags: KillFlag)
548 .addImm(Val: 0);
549 return;
550 }
551
552 // Handle copy from csr
553 if (RISCV::VCSRRegClass.contains(Reg: SrcReg) &&
554 RISCV::GPRRegClass.contains(Reg: DstReg)) {
555 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::CSRRS), DestReg: DstReg)
556 .addImm(Val: RISCVSysReg::lookupSysRegByName(Name: TRI->getName(RegNo: SrcReg))->Encoding)
557 .addReg(RegNo: RISCV::X0);
558 return;
559 }
560
561 if (RISCV::FPR16RegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
562 unsigned Opc;
563 if (STI.hasStdExtZfh()) {
564 Opc = RISCV::FSGNJ_H;
565 } else {
566 assert(STI.hasStdExtF() &&
567 (STI.hasStdExtZfhmin() || STI.hasStdExtZfbfmin()) &&
568 "Unexpected extensions");
569 // Zfhmin/Zfbfmin doesn't have FSGNJ_H, replace FSGNJ_H with FSGNJ_S.
570 DstReg = TRI->getMatchingSuperReg(Reg: DstReg, SubIdx: RISCV::sub_16,
571 RC: &RISCV::FPR32RegClass);
572 SrcReg = TRI->getMatchingSuperReg(Reg: SrcReg, SubIdx: RISCV::sub_16,
573 RC: &RISCV::FPR32RegClass);
574 Opc = RISCV::FSGNJ_S;
575 }
576 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: Opc), DestReg: DstReg)
577 .addReg(RegNo: SrcReg, Flags: KillFlag)
578 .addReg(RegNo: SrcReg, Flags: KillFlag);
579 return;
580 }
581
582 if (RISCV::FPR32RegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
583 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FSGNJ_S), DestReg: DstReg)
584 .addReg(RegNo: SrcReg, Flags: KillFlag)
585 .addReg(RegNo: SrcReg, Flags: KillFlag);
586 return;
587 }
588
589 if (RISCV::FPR64RegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
590 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FSGNJ_D), DestReg: DstReg)
591 .addReg(RegNo: SrcReg, Flags: KillFlag)
592 .addReg(RegNo: SrcReg, Flags: KillFlag);
593 return;
594 }
595
596 if (RISCV::FPR16RegClass.contains(Reg: DstReg) &&
597 RISCV::GPRRegClass.contains(Reg: SrcReg)) {
598 assert(STI.hasHalfFPLoadStoreMove() && "Unexpected extensions");
599 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FMV_H_X), DestReg: DstReg)
600 .addReg(RegNo: SrcReg, Flags: KillFlag);
601 return;
602 }
603
604 if (RISCV::GPRRegClass.contains(Reg: DstReg) &&
605 RISCV::FPR16RegClass.contains(Reg: SrcReg)) {
606 assert(STI.hasHalfFPLoadStoreMove() && "Unexpected extensions");
607 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FMV_X_H), DestReg: DstReg)
608 .addReg(RegNo: SrcReg, Flags: KillFlag);
609 return;
610 }
611
612 if (RISCV::FPR32RegClass.contains(Reg: DstReg) &&
613 RISCV::GPRRegClass.contains(Reg: SrcReg)) {
614 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FMV_W_X), DestReg: DstReg)
615 .addReg(RegNo: SrcReg, Flags: KillFlag);
616 return;
617 }
618
619 if (RISCV::GPRRegClass.contains(Reg: DstReg) &&
620 RISCV::FPR32RegClass.contains(Reg: SrcReg)) {
621 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FMV_X_W), DestReg: DstReg)
622 .addReg(RegNo: SrcReg, Flags: KillFlag);
623 return;
624 }
625
626 if (RISCV::FPR64RegClass.contains(Reg: DstReg) &&
627 RISCV::GPRRegClass.contains(Reg: SrcReg)) {
628 assert(STI.getXLen() == 64 && "Unexpected GPR size");
629 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FMV_D_X), DestReg: DstReg)
630 .addReg(RegNo: SrcReg, Flags: KillFlag);
631 return;
632 }
633
634 if (RISCV::GPRRegClass.contains(Reg: DstReg) &&
635 RISCV::FPR64RegClass.contains(Reg: SrcReg)) {
636 assert(STI.getXLen() == 64 && "Unexpected GPR size");
637 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FMV_X_D), DestReg: DstReg)
638 .addReg(RegNo: SrcReg, Flags: KillFlag);
639 return;
640 }
641
642 // VR->VR copies.
643 const TargetRegisterClass *RegClass =
644 TRI->getCommonMinimalPhysRegClass(Reg1: SrcReg, Reg2: DstReg);
645 if (RISCVRegisterInfo::isRVVRegClass(RC: RegClass)) {
646 copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RegClass);
647 return;
648 }
649
650 llvm_unreachable("Impossible reg-to-reg copy");
651}
652
653void RISCVInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
654 MachineBasicBlock::iterator I,
655 Register SrcReg, bool IsKill, int FI,
656 const TargetRegisterClass *RC,
657 Register VReg,
658 MachineInstr::MIFlag Flags) const {
659 MachineFunction *MF = MBB.getParent();
660 MachineFrameInfo &MFI = MF->getFrameInfo();
661 Align Alignment = MFI.getObjectAlign(ObjectIdx: FI);
662
663 unsigned Opcode;
664 if (RISCV::GPRRegClass.hasSubClassEq(RC)) {
665 Opcode = RegInfo.getRegSizeInBits(RC: RISCV::GPRRegClass) == 32 ? RISCV::SW
666 : RISCV::SD;
667 } else if (RISCV::GPRF16RegClass.hasSubClassEq(RC)) {
668 Opcode = RISCV::SH_INX;
669 } else if (RISCV::GPRF32RegClass.hasSubClassEq(RC)) {
670 Opcode = RISCV::SW_INX;
671 } else if (RISCV::GPRPairRegClass.hasSubClassEq(RC)) {
672 if (!STI.is64Bit() && STI.hasStdExtZilsd() &&
673 Alignment >= STI.getZilsdAlign()) {
674 Opcode = RISCV::SD_RV32;
675 } else {
676 Opcode = RISCV::PseudoRV32ZdinxSD;
677 }
678 } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) {
679 Opcode = RISCV::FSH;
680 } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) {
681 Opcode = RISCV::FSW;
682 } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) {
683 Opcode = RISCV::FSD;
684 } else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
685 Opcode = RISCV::VS1R_V;
686 } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
687 Opcode = RISCV::VS2R_V;
688 } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
689 Opcode = RISCV::VS4R_V;
690 } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
691 Opcode = RISCV::VS8R_V;
692 } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC))
693 Opcode = RISCV::PseudoVSPILL2_M1;
694 else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC))
695 Opcode = RISCV::PseudoVSPILL2_M2;
696 else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC))
697 Opcode = RISCV::PseudoVSPILL2_M4;
698 else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC))
699 Opcode = RISCV::PseudoVSPILL3_M1;
700 else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC))
701 Opcode = RISCV::PseudoVSPILL3_M2;
702 else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC))
703 Opcode = RISCV::PseudoVSPILL4_M1;
704 else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC))
705 Opcode = RISCV::PseudoVSPILL4_M2;
706 else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC))
707 Opcode = RISCV::PseudoVSPILL5_M1;
708 else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC))
709 Opcode = RISCV::PseudoVSPILL6_M1;
710 else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC))
711 Opcode = RISCV::PseudoVSPILL7_M1;
712 else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC))
713 Opcode = RISCV::PseudoVSPILL8_M1;
714 else
715 llvm_unreachable("Can't store this register to stack slot");
716
717 if (RISCVRegisterInfo::isRVVRegClass(RC)) {
718 MachineMemOperand *MMO = MF->getMachineMemOperand(
719 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), F: MachineMemOperand::MOStore,
720 Size: TypeSize::getScalable(MinimumSize: MFI.getObjectSize(ObjectIdx: FI)), BaseAlignment: Alignment);
721
722 MFI.setStackID(ObjectIdx: FI, ID: TargetStackID::ScalableVector);
723 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode))
724 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: IsKill))
725 .addFrameIndex(Idx: FI)
726 .addMemOperand(MMO)
727 .setMIFlag(Flags);
728 NumVRegSpilled += RegInfo.getRegSizeInBits(RC: *RC) / RISCV::RVVBitsPerBlock;
729 } else {
730 MachineMemOperand *MMO = MF->getMachineMemOperand(
731 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), F: MachineMemOperand::MOStore,
732 Size: MFI.getObjectSize(ObjectIdx: FI), BaseAlignment: Alignment);
733
734 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode))
735 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: IsKill))
736 .addFrameIndex(Idx: FI)
737 .addImm(Val: 0)
738 .addMemOperand(MMO)
739 .setMIFlag(Flags);
740 }
741}
742
743void RISCVInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
744 MachineBasicBlock::iterator I,
745 Register DstReg, int FI,
746 const TargetRegisterClass *RC,
747 Register VReg, unsigned SubReg,
748 MachineInstr::MIFlag Flags) const {
749 MachineFunction *MF = MBB.getParent();
750 MachineFrameInfo &MFI = MF->getFrameInfo();
751 Align Alignment = MFI.getObjectAlign(ObjectIdx: FI);
752 DebugLoc DL =
753 Flags & MachineInstr::FrameDestroy ? MBB.findDebugLoc(MBBI: I) : DebugLoc();
754
755 unsigned Opcode;
756 if (RISCV::GPRRegClass.hasSubClassEq(RC)) {
757 Opcode = RegInfo.getRegSizeInBits(RC: RISCV::GPRRegClass) == 32 ? RISCV::LW
758 : RISCV::LD;
759 } else if (RISCV::GPRF16RegClass.hasSubClassEq(RC)) {
760 Opcode = RISCV::LH_INX;
761 } else if (RISCV::GPRF32RegClass.hasSubClassEq(RC)) {
762 Opcode = RISCV::LW_INX;
763 } else if (RISCV::GPRPairRegClass.hasSubClassEq(RC)) {
764 if (!STI.is64Bit() && STI.hasStdExtZilsd() &&
765 Alignment >= STI.getZilsdAlign()) {
766 Opcode = RISCV::LD_RV32;
767 } else {
768 Opcode = RISCV::PseudoRV32ZdinxLD;
769 }
770 } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) {
771 Opcode = RISCV::FLH;
772 } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) {
773 Opcode = RISCV::FLW;
774 } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) {
775 Opcode = RISCV::FLD;
776 } else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
777 Opcode = RISCV::VL1RE8_V;
778 } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
779 Opcode = RISCV::VL2RE8_V;
780 } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
781 Opcode = RISCV::VL4RE8_V;
782 } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
783 Opcode = RISCV::VL8RE8_V;
784 } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC))
785 Opcode = RISCV::PseudoVRELOAD2_M1;
786 else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC))
787 Opcode = RISCV::PseudoVRELOAD2_M2;
788 else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC))
789 Opcode = RISCV::PseudoVRELOAD2_M4;
790 else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC))
791 Opcode = RISCV::PseudoVRELOAD3_M1;
792 else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC))
793 Opcode = RISCV::PseudoVRELOAD3_M2;
794 else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC))
795 Opcode = RISCV::PseudoVRELOAD4_M1;
796 else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC))
797 Opcode = RISCV::PseudoVRELOAD4_M2;
798 else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC))
799 Opcode = RISCV::PseudoVRELOAD5_M1;
800 else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC))
801 Opcode = RISCV::PseudoVRELOAD6_M1;
802 else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC))
803 Opcode = RISCV::PseudoVRELOAD7_M1;
804 else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC))
805 Opcode = RISCV::PseudoVRELOAD8_M1;
806 else
807 llvm_unreachable("Can't load this register from stack slot");
808
809 if (RISCVRegisterInfo::isRVVRegClass(RC)) {
810 MachineMemOperand *MMO = MF->getMachineMemOperand(
811 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), F: MachineMemOperand::MOLoad,
812 Size: TypeSize::getScalable(MinimumSize: MFI.getObjectSize(ObjectIdx: FI)), BaseAlignment: Alignment);
813
814 MFI.setStackID(ObjectIdx: FI, ID: TargetStackID::ScalableVector);
815 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode), DestReg: DstReg)
816 .addFrameIndex(Idx: FI)
817 .addMemOperand(MMO)
818 .setMIFlag(Flags);
819 NumVRegReloaded += RegInfo.getRegSizeInBits(RC: *RC) / RISCV::RVVBitsPerBlock;
820 } else {
821 MachineMemOperand *MMO = MF->getMachineMemOperand(
822 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), F: MachineMemOperand::MOLoad,
823 Size: MFI.getObjectSize(ObjectIdx: FI), BaseAlignment: Alignment);
824
825 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode), DestReg: DstReg)
826 .addFrameIndex(Idx: FI)
827 .addImm(Val: 0)
828 .addMemOperand(MMO)
829 .setMIFlag(Flags);
830 }
831}
832std::optional<unsigned> getFoldedOpcode(MachineFunction &MF, MachineInstr &MI,
833 ArrayRef<unsigned> Ops,
834 const RISCVSubtarget &ST) {
835
836 // The below optimizations narrow the load so they are only valid for little
837 // endian.
838 // TODO: Support big endian by adding an offset into the frame object?
839 if (MF.getDataLayout().isBigEndian())
840 return std::nullopt;
841
842 // Fold load from stack followed by sext.b/sext.h/sext.w/zext.b/zext.h/zext.w.
843 if (Ops.size() != 1 || Ops[0] != 1)
844 return std::nullopt;
845
846 switch (MI.getOpcode()) {
847 default:
848 if (RISCVInstrInfo::isSEXT_W(MI))
849 return RISCV::LW;
850 if (RISCVInstrInfo::isZEXT_W(MI))
851 return RISCV::LWU;
852 if (RISCVInstrInfo::isZEXT_B(MI))
853 return RISCV::LBU;
854 break;
855 case RISCV::SEXT_H:
856 return RISCV::LH;
857 case RISCV::SEXT_B:
858 return RISCV::LB;
859 case RISCV::ZEXT_H_RV32:
860 case RISCV::ZEXT_H_RV64:
861 return RISCV::LHU;
862 // A GPR reload feeding an int->FP move can be an FP load from the slot.
863 case RISCV::FMV_H_X:
864 return RISCV::FLH;
865 case RISCV::FMV_W_X:
866 return RISCV::FLW;
867 case RISCV::FMV_D_X:
868 return RISCV::FLD;
869 }
870
871 switch (RISCV::getRVVMCOpcode(RVVPseudoOpcode: MI.getOpcode())) {
872 default:
873 return std::nullopt;
874 case RISCV::VMV_X_S: {
875 unsigned Log2SEW =
876 MI.getOperand(i: RISCVII::getSEWOpNum(Desc: MI.getDesc())).getImm();
877 if (ST.getXLen() < (1U << Log2SEW))
878 return std::nullopt;
879 switch (Log2SEW) {
880 case 3:
881 return RISCV::LB;
882 case 4:
883 return RISCV::LH;
884 case 5:
885 return RISCV::LW;
886 case 6:
887 return RISCV::LD;
888 default:
889 llvm_unreachable("Unexpected SEW");
890 }
891 }
892 case RISCV::VFMV_F_S: {
893 unsigned Log2SEW =
894 MI.getOperand(i: RISCVII::getSEWOpNum(Desc: MI.getDesc())).getImm();
895 switch (Log2SEW) {
896 case 4:
897 return RISCV::FLH;
898 case 5:
899 return RISCV::FLW;
900 case 6:
901 return RISCV::FLD;
902 default:
903 llvm_unreachable("Unexpected SEW");
904 }
905 }
906 }
907}
908
909// This is the version used during InlineSpiller::spillAroundUses
910MachineInstr *
911RISCVInstrInfo::foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
912 ArrayRef<unsigned> Ops, int FrameIndex,
913 MachineInstr *&CopyMI, LiveIntervals *LIS,
914 VirtRegMap *VRM) const {
915 MachineBasicBlock::iterator InsertPt = MI;
916 std::optional<unsigned> LoadOpc = getFoldedOpcode(MF, MI, Ops, ST: STI);
917 if (!LoadOpc)
918 return nullptr;
919 Register DstReg = MI.getOperand(i: 0).getReg();
920 return BuildMI(BB&: *MI.getParent(), I: InsertPt, MIMD: MI.getDebugLoc(), MCID: get(Opcode: *LoadOpc),
921 DestReg: DstReg)
922 .addFrameIndex(Idx: FrameIndex)
923 .addImm(Val: 0);
924}
925
926static unsigned getLoadPredicatedOpcode(unsigned Opcode) {
927 switch (Opcode) {
928 case RISCV::LB:
929 return RISCV::PseudoCCLB;
930 case RISCV::LBU:
931 return RISCV::PseudoCCLBU;
932 case RISCV::LH:
933 return RISCV::PseudoCCLH;
934 case RISCV::LHU:
935 return RISCV::PseudoCCLHU;
936 case RISCV::LW:
937 return RISCV::PseudoCCLW;
938 case RISCV::LWU:
939 return RISCV::PseudoCCLWU;
940 case RISCV::LD:
941 return RISCV::PseudoCCLD;
942 case RISCV::QC_E_LB:
943 return RISCV::PseudoCCQC_E_LB;
944 case RISCV::QC_E_LBU:
945 return RISCV::PseudoCCQC_E_LBU;
946 case RISCV::QC_E_LH:
947 return RISCV::PseudoCCQC_E_LH;
948 case RISCV::QC_E_LHU:
949 return RISCV::PseudoCCQC_E_LHU;
950 case RISCV::QC_E_LW:
951 return RISCV::PseudoCCQC_E_LW;
952 default:
953 return 0;
954 }
955}
956
957MachineInstr *RISCVInstrInfo::foldMemoryOperandImpl(
958 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
959 MachineInstr &LoadMI, MachineInstr *&CopyMI, LiveIntervals *LIS,
960 VirtRegMap *VRM) const {
961 MachineBasicBlock::iterator InsertPt = MI;
962 // For now, only handle RISCV::PseudoCCMOVGPR.
963 if (MI.getOpcode() != RISCV::PseudoCCMOVGPR)
964 return nullptr;
965
966 unsigned PredOpc = getLoadPredicatedOpcode(Opcode: LoadMI.getOpcode());
967
968 if (!STI.hasShortForwardBranchILoad() || !PredOpc)
969 return nullptr;
970
971 MachineRegisterInfo &MRI = MF.getRegInfo();
972 if (Ops.size() != 1 || (Ops[0] != 1 && Ops[0] != 2))
973 return nullptr;
974
975 bool Invert = Ops[0] == 2;
976 const MachineOperand &FalseReg = MI.getOperand(i: !Invert ? 2 : 1);
977 Register DestReg = MI.getOperand(i: 0).getReg();
978 const TargetRegisterClass *PreviousClass = MRI.getRegClass(Reg: FalseReg.getReg());
979 if (!MRI.constrainRegClass(Reg: DestReg, RC: PreviousClass))
980 return nullptr;
981
982 // Create a new predicated version of DefMI.
983 MachineInstrBuilder NewMI = BuildMI(BB&: *MI.getParent(), I: InsertPt,
984 MIMD: MI.getDebugLoc(), MCID: get(Opcode: PredOpc), DestReg);
985
986 // Copy the false register.
987 NewMI.add(MO: FalseReg);
988
989 // Copy all the DefMI operands.
990 const MCInstrDesc &DefDesc = LoadMI.getDesc();
991 for (unsigned i = 1, e = DefDesc.getNumOperands(); i != e; ++i)
992 NewMI.add(MO: LoadMI.getOperand(i));
993
994 // Add branch opcode, inverting if necessary.
995 unsigned BCC = MI.getOperand(i: MI.getNumExplicitOperands() - 3).getImm();
996 if (!Invert)
997 BCC = RISCVCC::getInverseBranchOpcode(BCC);
998 NewMI.addImm(Val: BCC);
999
1000 // Copy condition portion
1001 NewMI.add(MOs: {MI.getOperand(i: MI.getNumExplicitOperands() - 2),
1002 MI.getOperand(i: MI.getNumExplicitOperands() - 1)});
1003 NewMI.cloneMemRefs(OtherMI: LoadMI);
1004 return NewMI;
1005}
1006
1007void RISCVInstrInfo::movImm(MachineBasicBlock &MBB,
1008 MachineBasicBlock::iterator MBBI,
1009 const DebugLoc &DL, Register DstReg, uint64_t Val,
1010 MachineInstr::MIFlag Flag, bool DstRenamable,
1011 bool DstIsDead) const {
1012 Register SrcReg = RISCV::X0;
1013
1014 // For RV32, allow a sign or unsigned 32 bit value.
1015 if (!STI.is64Bit() && !isInt<32>(x: Val)) {
1016 // If have a uimm32 it will still fit in a register so we can allow it.
1017 if (!isUInt<32>(x: Val))
1018 report_fatal_error(reason: "Should only materialize 32-bit constants for RV32");
1019
1020 // Sign extend for generateInstSeq.
1021 Val = SignExtend64<32>(x: Val);
1022 }
1023
1024 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val, STI);
1025 assert(!Seq.empty());
1026
1027 bool SrcRenamable = false;
1028 unsigned Num = 0;
1029
1030 for (const RISCVMatInt::Inst &Inst : Seq) {
1031 bool LastItem = ++Num == Seq.size();
1032 RegState DstRegState = getDeadRegState(B: DstIsDead && LastItem) |
1033 getRenamableRegState(B: DstRenamable);
1034 RegState SrcRegState = getKillRegState(B: SrcReg != RISCV::X0) |
1035 getRenamableRegState(B: SrcRenamable);
1036 switch (Inst.getOpndKind()) {
1037 case RISCVMatInt::Imm:
1038 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: Inst.getOpcode()))
1039 .addDef(RegNo: DstReg, Flags: DstRegState)
1040 .addImm(Val: Inst.getImm())
1041 .setMIFlag(Flag);
1042 break;
1043 case RISCVMatInt::RegX0:
1044 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: Inst.getOpcode()))
1045 .addDef(RegNo: DstReg, Flags: DstRegState)
1046 .addReg(RegNo: SrcReg, Flags: SrcRegState)
1047 .addReg(RegNo: RISCV::X0)
1048 .setMIFlag(Flag);
1049 break;
1050 case RISCVMatInt::RegReg:
1051 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: Inst.getOpcode()))
1052 .addDef(RegNo: DstReg, Flags: DstRegState)
1053 .addReg(RegNo: SrcReg, Flags: SrcRegState)
1054 .addReg(RegNo: SrcReg, Flags: SrcRegState)
1055 .setMIFlag(Flag);
1056 break;
1057 case RISCVMatInt::RegImm:
1058 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: Inst.getOpcode()))
1059 .addDef(RegNo: DstReg, Flags: DstRegState)
1060 .addReg(RegNo: SrcReg, Flags: SrcRegState)
1061 .addImm(Val: Inst.getImm())
1062 .setMIFlag(Flag);
1063 break;
1064 }
1065
1066 // Only the first instruction has X0 as its source.
1067 SrcReg = DstReg;
1068 SrcRenamable = DstRenamable;
1069 }
1070}
1071
1072RISCVCC::CondCode RISCVInstrInfo::getCondFromBranchOpc(unsigned Opc) {
1073 switch (Opc) {
1074 default:
1075 return RISCVCC::COND_INVALID;
1076 case RISCV::BEQ:
1077 case RISCV::BEQI:
1078 case RISCV::CV_BEQIMM:
1079 case RISCV::QC_BEQI:
1080 case RISCV::QC_E_BEQI:
1081 case RISCV::NDS_BBC:
1082 case RISCV::NDS_BEQC:
1083 return RISCVCC::COND_EQ;
1084 case RISCV::BNE:
1085 case RISCV::BNEI:
1086 case RISCV::QC_BNEI:
1087 case RISCV::QC_E_BNEI:
1088 case RISCV::CV_BNEIMM:
1089 case RISCV::NDS_BBS:
1090 case RISCV::NDS_BNEC:
1091 return RISCVCC::COND_NE;
1092 case RISCV::BLT:
1093 case RISCV::QC_BLTI:
1094 case RISCV::QC_E_BLTI:
1095 return RISCVCC::COND_LT;
1096 case RISCV::BGE:
1097 case RISCV::QC_BGEI:
1098 case RISCV::QC_E_BGEI:
1099 return RISCVCC::COND_GE;
1100 case RISCV::BLTU:
1101 case RISCV::QC_BLTUI:
1102 case RISCV::QC_E_BLTUI:
1103 return RISCVCC::COND_LTU;
1104 case RISCV::BGEU:
1105 case RISCV::QC_BGEUI:
1106 case RISCV::QC_E_BGEUI:
1107 return RISCVCC::COND_GEU;
1108 }
1109}
1110
1111bool RISCVInstrInfo::evaluateCondBranch(RISCVCC::CondCode CC, int64_t C0,
1112 int64_t C1) {
1113 switch (CC) {
1114 default:
1115 llvm_unreachable("Unexpected CC");
1116 case RISCVCC::COND_EQ:
1117 return C0 == C1;
1118 case RISCVCC::COND_NE:
1119 return C0 != C1;
1120 case RISCVCC::COND_LT:
1121 return C0 < C1;
1122 case RISCVCC::COND_GE:
1123 return C0 >= C1;
1124 case RISCVCC::COND_LTU:
1125 return (uint64_t)C0 < (uint64_t)C1;
1126 case RISCVCC::COND_GEU:
1127 return (uint64_t)C0 >= (uint64_t)C1;
1128 }
1129}
1130
1131// The contents of values added to Cond are not examined outside of
1132// RISCVInstrInfo, giving us flexibility in what to push to it. For RISCV, we
1133// push BranchOpcode, Reg1, Reg2.
1134static void parseCondBranch(MachineInstr &LastInst, MachineBasicBlock *&Target,
1135 SmallVectorImpl<MachineOperand> &Cond) {
1136 // Block ends with fall-through condbranch.
1137 assert(LastInst.getDesc().isConditionalBranch() &&
1138 "Unknown conditional branch");
1139 Target = LastInst.getOperand(i: 2).getMBB();
1140 Cond.push_back(Elt: MachineOperand::CreateImm(Val: LastInst.getOpcode()));
1141 Cond.push_back(Elt: LastInst.getOperand(i: 0));
1142 Cond.push_back(Elt: LastInst.getOperand(i: 1));
1143}
1144
1145static unsigned getInverseXqcicmOpcode(unsigned Opcode) {
1146 switch (Opcode) {
1147 default:
1148 llvm_unreachable("Unexpected Opcode");
1149 case RISCV::QC_MVEQ:
1150 return RISCV::QC_MVNE;
1151 case RISCV::QC_MVNE:
1152 return RISCV::QC_MVEQ;
1153 case RISCV::QC_MVLT:
1154 return RISCV::QC_MVGE;
1155 case RISCV::QC_MVGE:
1156 return RISCV::QC_MVLT;
1157 case RISCV::QC_MVLTU:
1158 return RISCV::QC_MVGEU;
1159 case RISCV::QC_MVGEU:
1160 return RISCV::QC_MVLTU;
1161 case RISCV::QC_MVEQI:
1162 return RISCV::QC_MVNEI;
1163 case RISCV::QC_MVNEI:
1164 return RISCV::QC_MVEQI;
1165 case RISCV::QC_MVLTI:
1166 return RISCV::QC_MVGEI;
1167 case RISCV::QC_MVGEI:
1168 return RISCV::QC_MVLTI;
1169 case RISCV::QC_MVLTUI:
1170 return RISCV::QC_MVGEUI;
1171 case RISCV::QC_MVGEUI:
1172 return RISCV::QC_MVLTUI;
1173 }
1174}
1175
1176unsigned RISCVCC::getBrCond(RISCVCC::CondCode CC, unsigned SelectOpc) {
1177 switch (SelectOpc) {
1178 default:
1179 switch (CC) {
1180 default:
1181 llvm_unreachable("Unexpected condition code!");
1182 case RISCVCC::COND_EQ:
1183 return RISCV::BEQ;
1184 case RISCVCC::COND_NE:
1185 return RISCV::BNE;
1186 case RISCVCC::COND_LT:
1187 return RISCV::BLT;
1188 case RISCVCC::COND_GE:
1189 return RISCV::BGE;
1190 case RISCVCC::COND_LTU:
1191 return RISCV::BLTU;
1192 case RISCVCC::COND_GEU:
1193 return RISCV::BGEU;
1194 }
1195 break;
1196 case RISCV::Select_GPR_Using_CC_Imm5_Zibi:
1197 switch (CC) {
1198 default:
1199 llvm_unreachable("Unexpected condition code!");
1200 case RISCVCC::COND_EQ:
1201 return RISCV::BEQI;
1202 case RISCVCC::COND_NE:
1203 return RISCV::BNEI;
1204 }
1205 break;
1206 case RISCV::Select_GPR_Using_CC_SImm5_CV:
1207 switch (CC) {
1208 default:
1209 llvm_unreachable("Unexpected condition code!");
1210 case RISCVCC::COND_EQ:
1211 return RISCV::CV_BEQIMM;
1212 case RISCVCC::COND_NE:
1213 return RISCV::CV_BNEIMM;
1214 }
1215 break;
1216 case RISCV::Select_GPRNoX0_Using_CC_SImm5NonZero_QC:
1217 switch (CC) {
1218 default:
1219 llvm_unreachable("Unexpected condition code!");
1220 case RISCVCC::COND_EQ:
1221 return RISCV::QC_BEQI;
1222 case RISCVCC::COND_NE:
1223 return RISCV::QC_BNEI;
1224 case RISCVCC::COND_LT:
1225 return RISCV::QC_BLTI;
1226 case RISCVCC::COND_GE:
1227 return RISCV::QC_BGEI;
1228 }
1229 break;
1230 case RISCV::Select_GPRNoX0_Using_CC_UImm5NonZero_QC:
1231 switch (CC) {
1232 default:
1233 llvm_unreachable("Unexpected condition code!");
1234 case RISCVCC::COND_LTU:
1235 return RISCV::QC_BLTUI;
1236 case RISCVCC::COND_GEU:
1237 return RISCV::QC_BGEUI;
1238 }
1239 break;
1240 case RISCV::Select_GPRNoX0_Using_CC_SImm16NonZero_QC:
1241 switch (CC) {
1242 default:
1243 llvm_unreachable("Unexpected condition code!");
1244 case RISCVCC::COND_EQ:
1245 return RISCV::QC_E_BEQI;
1246 case RISCVCC::COND_NE:
1247 return RISCV::QC_E_BNEI;
1248 case RISCVCC::COND_LT:
1249 return RISCV::QC_E_BLTI;
1250 case RISCVCC::COND_GE:
1251 return RISCV::QC_E_BGEI;
1252 }
1253 break;
1254 case RISCV::Select_GPRNoX0_Using_CC_UImm16NonZero_QC:
1255 switch (CC) {
1256 default:
1257 llvm_unreachable("Unexpected condition code!");
1258 case RISCVCC::COND_LTU:
1259 return RISCV::QC_E_BLTUI;
1260 case RISCVCC::COND_GEU:
1261 return RISCV::QC_E_BGEUI;
1262 }
1263 break;
1264 case RISCV::Select_GPR_Using_CC_UImmLog2XLen_NDS:
1265 switch (CC) {
1266 default:
1267 llvm_unreachable("Unexpected condition code!");
1268 case RISCVCC::COND_EQ:
1269 return RISCV::NDS_BBC;
1270 case RISCVCC::COND_NE:
1271 return RISCV::NDS_BBS;
1272 }
1273 break;
1274 case RISCV::Select_GPR_Using_CC_UImm7_NDS:
1275 switch (CC) {
1276 default:
1277 llvm_unreachable("Unexpected condition code!");
1278 case RISCVCC::COND_EQ:
1279 return RISCV::NDS_BEQC;
1280 case RISCVCC::COND_NE:
1281 return RISCV::NDS_BNEC;
1282 }
1283 break;
1284 }
1285}
1286
1287RISCVCC::CondCode RISCVCC::getInverseBranchCondition(RISCVCC::CondCode CC) {
1288 switch (CC) {
1289 default:
1290 llvm_unreachable("Unrecognized conditional branch");
1291 case RISCVCC::COND_EQ:
1292 return RISCVCC::COND_NE;
1293 case RISCVCC::COND_NE:
1294 return RISCVCC::COND_EQ;
1295 case RISCVCC::COND_LT:
1296 return RISCVCC::COND_GE;
1297 case RISCVCC::COND_GE:
1298 return RISCVCC::COND_LT;
1299 case RISCVCC::COND_LTU:
1300 return RISCVCC::COND_GEU;
1301 case RISCVCC::COND_GEU:
1302 return RISCVCC::COND_LTU;
1303 }
1304}
1305
1306// Return inverse branch
1307unsigned RISCVCC::getInverseBranchOpcode(unsigned BCC) {
1308 switch (BCC) {
1309 default:
1310 llvm_unreachable("Unexpected branch opcode!");
1311 case RISCV::BEQ:
1312 return RISCV::BNE;
1313 case RISCV::BEQI:
1314 return RISCV::BNEI;
1315 case RISCV::BNE:
1316 return RISCV::BEQ;
1317 case RISCV::BNEI:
1318 return RISCV::BEQI;
1319 case RISCV::BLT:
1320 return RISCV::BGE;
1321 case RISCV::BGE:
1322 return RISCV::BLT;
1323 case RISCV::BLTU:
1324 return RISCV::BGEU;
1325 case RISCV::BGEU:
1326 return RISCV::BLTU;
1327 case RISCV::CV_BEQIMM:
1328 return RISCV::CV_BNEIMM;
1329 case RISCV::CV_BNEIMM:
1330 return RISCV::CV_BEQIMM;
1331 case RISCV::QC_BEQI:
1332 return RISCV::QC_BNEI;
1333 case RISCV::QC_BNEI:
1334 return RISCV::QC_BEQI;
1335 case RISCV::QC_BLTI:
1336 return RISCV::QC_BGEI;
1337 case RISCV::QC_BGEI:
1338 return RISCV::QC_BLTI;
1339 case RISCV::QC_BLTUI:
1340 return RISCV::QC_BGEUI;
1341 case RISCV::QC_BGEUI:
1342 return RISCV::QC_BLTUI;
1343 case RISCV::QC_E_BEQI:
1344 return RISCV::QC_E_BNEI;
1345 case RISCV::QC_E_BNEI:
1346 return RISCV::QC_E_BEQI;
1347 case RISCV::QC_E_BLTI:
1348 return RISCV::QC_E_BGEI;
1349 case RISCV::QC_E_BGEI:
1350 return RISCV::QC_E_BLTI;
1351 case RISCV::QC_E_BLTUI:
1352 return RISCV::QC_E_BGEUI;
1353 case RISCV::QC_E_BGEUI:
1354 return RISCV::QC_E_BLTUI;
1355 case RISCV::NDS_BBC:
1356 return RISCV::NDS_BBS;
1357 case RISCV::NDS_BBS:
1358 return RISCV::NDS_BBC;
1359 case RISCV::NDS_BEQC:
1360 return RISCV::NDS_BNEC;
1361 case RISCV::NDS_BNEC:
1362 return RISCV::NDS_BEQC;
1363 }
1364}
1365
1366bool RISCVInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
1367 MachineBasicBlock *&TBB,
1368 MachineBasicBlock *&FBB,
1369 SmallVectorImpl<MachineOperand> &Cond,
1370 bool AllowModify) const {
1371 TBB = FBB = nullptr;
1372 Cond.clear();
1373
1374 // If the block has no terminators, it just falls into the block after it.
1375 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
1376 if (I == MBB.end() || !isUnpredicatedTerminator(MI: *I))
1377 return false;
1378
1379 // Count the number of terminators and find the first unconditional or
1380 // indirect branch.
1381 MachineBasicBlock::iterator FirstUncondOrIndirectBr = MBB.end();
1382 int NumTerminators = 0;
1383 for (auto J = I.getReverse(); J != MBB.rend() && isUnpredicatedTerminator(MI: *J);
1384 J++) {
1385 NumTerminators++;
1386 if (J->getDesc().isUnconditionalBranch() ||
1387 J->getDesc().isIndirectBranch()) {
1388 FirstUncondOrIndirectBr = J.getReverse();
1389 }
1390 }
1391
1392 // If AllowModify is true, we can erase any terminators after
1393 // FirstUncondOrIndirectBR.
1394 if (AllowModify && FirstUncondOrIndirectBr != MBB.end()) {
1395 while (std::next(x: FirstUncondOrIndirectBr) != MBB.end()) {
1396 std::next(x: FirstUncondOrIndirectBr)->eraseFromParent();
1397 NumTerminators--;
1398 }
1399 I = FirstUncondOrIndirectBr;
1400 }
1401
1402 // We can't handle blocks that end in an indirect branch.
1403 if (I->getDesc().isIndirectBranch())
1404 return true;
1405
1406 // We can't handle Generic branch opcodes from Global ISel.
1407 if (I->isPreISelOpcode())
1408 return true;
1409
1410 // We can't handle blocks with more than 2 terminators.
1411 if (NumTerminators > 2)
1412 return true;
1413
1414 // Handle a single unconditional branch.
1415 if (NumTerminators == 1 && I->getDesc().isUnconditionalBranch()) {
1416 TBB = getBranchDestBlock(MI: *I);
1417 return false;
1418 }
1419
1420 // Handle a single conditional branch.
1421 if (NumTerminators == 1 && I->getDesc().isConditionalBranch()) {
1422 parseCondBranch(LastInst&: *I, Target&: TBB, Cond);
1423 return false;
1424 }
1425
1426 // Handle a conditional branch followed by an unconditional branch.
1427 if (NumTerminators == 2 && std::prev(x: I)->getDesc().isConditionalBranch() &&
1428 I->getDesc().isUnconditionalBranch()) {
1429 parseCondBranch(LastInst&: *std::prev(x: I), Target&: TBB, Cond);
1430 FBB = getBranchDestBlock(MI: *I);
1431 return false;
1432 }
1433
1434 // Otherwise, we can't handle this.
1435 return true;
1436}
1437
1438unsigned RISCVInstrInfo::removeBranch(MachineBasicBlock &MBB,
1439 int *BytesRemoved) const {
1440 if (BytesRemoved)
1441 *BytesRemoved = 0;
1442 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
1443 if (I == MBB.end())
1444 return 0;
1445
1446 if (!I->getDesc().isUnconditionalBranch() &&
1447 !I->getDesc().isConditionalBranch())
1448 return 0;
1449
1450 // Remove the branch.
1451 if (BytesRemoved)
1452 *BytesRemoved += getInstSizeInBytes(MI: *I);
1453 I->eraseFromParent();
1454
1455 I = MBB.end();
1456
1457 if (I == MBB.begin())
1458 return 1;
1459 --I;
1460 if (!I->getDesc().isConditionalBranch())
1461 return 1;
1462
1463 // Remove the branch.
1464 if (BytesRemoved)
1465 *BytesRemoved += getInstSizeInBytes(MI: *I);
1466 I->eraseFromParent();
1467 return 2;
1468}
1469
1470// Inserts a branch into the end of the specific MachineBasicBlock, returning
1471// the number of instructions inserted.
1472unsigned RISCVInstrInfo::insertBranch(
1473 MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
1474 ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const {
1475 if (BytesAdded)
1476 *BytesAdded = 0;
1477
1478 // Shouldn't be a fall through.
1479 assert(TBB && "insertBranch must not be told to insert a fallthrough");
1480 assert((Cond.size() == 3 || Cond.size() == 0) &&
1481 "RISC-V branch conditions have two components!");
1482
1483 // Unconditional branch.
1484 if (Cond.empty()) {
1485 MachineInstr &MI = *BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: RISCV::PseudoBR)).addMBB(MBB: TBB);
1486 if (BytesAdded)
1487 *BytesAdded += getInstSizeInBytes(MI);
1488 return 1;
1489 }
1490
1491 // Either a one or two-way conditional branch.
1492 MachineInstr &CondMI = *BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: Cond[0].getImm()))
1493 .add(MO: Cond[1])
1494 .add(MO: Cond[2])
1495 .addMBB(MBB: TBB);
1496 if (BytesAdded)
1497 *BytesAdded += getInstSizeInBytes(MI: CondMI);
1498
1499 // One-way conditional branch.
1500 if (!FBB)
1501 return 1;
1502
1503 // Two-way conditional branch.
1504 MachineInstr &MI = *BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: RISCV::PseudoBR)).addMBB(MBB: FBB);
1505 if (BytesAdded)
1506 *BytesAdded += getInstSizeInBytes(MI);
1507 return 2;
1508}
1509
1510void RISCVInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
1511 MachineBasicBlock &DestBB,
1512 MachineBasicBlock &RestoreBB,
1513 const DebugLoc &DL, int64_t BrOffset,
1514 RegScavenger *RS) const {
1515 assert(RS && "RegScavenger required for long branching");
1516 assert(MBB.empty() &&
1517 "new block should be inserted for expanding unconditional branch");
1518 assert(MBB.pred_size() == 1);
1519 assert(RestoreBB.empty() &&
1520 "restore block should be inserted for restoring clobbered registers");
1521
1522 MachineFunction *MF = MBB.getParent();
1523 MachineRegisterInfo &MRI = MF->getRegInfo();
1524 RISCVMachineFunctionInfo *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
1525 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
1526
1527 if (!isInt<32>(x: BrOffset))
1528 report_fatal_error(
1529 reason: "Branch offsets outside of the signed 32-bit range not supported");
1530
1531 // FIXME: A virtual register must be used initially, as the register
1532 // scavenger won't work with empty blocks (SIInstrInfo::insertIndirectBranch
1533 // uses the same workaround).
1534 Register ScratchReg = MRI.createVirtualRegister(RegClass: &RISCV::GPRJALRRegClass);
1535 auto II = MBB.end();
1536 // We may also update the jump target to RestoreBB later.
1537 MachineInstr &MI = *BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::PseudoJump))
1538 .addDef(RegNo: ScratchReg, Flags: RegState::Dead)
1539 .addMBB(MBB: &DestBB, TargetFlags: RISCVII::MO_CALL);
1540
1541 RS->enterBasicBlockEnd(MBB);
1542 // When cf-protection-branch is enabled, we must use t2 (x7) for software
1543 // guarded branches to hold the landing pad label.
1544 bool HasCFBranch =
1545 MF->getInfo<RISCVMachineFunctionInfo>()->hasCFProtectionBranch();
1546 const TargetRegisterClass *RC = &RISCV::GPRRegClass;
1547 if (HasCFBranch)
1548 RC = &RISCV::GPRX7RegClass;
1549 Register TmpGPR =
1550 RS->scavengeRegisterBackwards(RC: *RC, To: MI.getIterator(),
1551 /*RestoreAfter=*/false, /*SpAdj=*/SPAdj: 0,
1552 /*AllowSpill=*/false);
1553 if (TmpGPR.isValid())
1554 RS->setRegUsed(Reg: TmpGPR);
1555 else {
1556 // The case when there is no scavenged register needs special handling.
1557
1558 // Pick s11(or s1 for rve) because it doesn't make a difference.
1559 TmpGPR = STI.hasStdExtE() ? RISCV::X9 : RISCV::X27;
1560 // Force t2 if cf-protection-branch is enabled
1561 if (HasCFBranch)
1562 TmpGPR = RISCV::X7;
1563
1564 int FrameIndex = RVFI->getBranchRelaxationScratchFrameIndex();
1565 if (FrameIndex == -1)
1566 report_fatal_error(reason: "underestimated function size");
1567
1568 storeRegToStackSlot(MBB, I: MI, SrcReg: TmpGPR, /*IsKill=*/true, FI: FrameIndex,
1569 RC: &RISCV::GPRRegClass, VReg: Register());
1570 TRI->eliminateFrameIndex(MI: std::prev(x: MI.getIterator()),
1571 /*SpAdj=*/SPAdj: 0, /*FIOperandNum=*/1);
1572
1573 MI.getOperand(i: 1).setMBB(&RestoreBB);
1574
1575 loadRegFromStackSlot(MBB&: RestoreBB, I: RestoreBB.end(), DstReg: TmpGPR, FI: FrameIndex,
1576 RC: &RISCV::GPRRegClass, VReg: Register());
1577 TRI->eliminateFrameIndex(MI: RestoreBB.back(),
1578 /*SpAdj=*/SPAdj: 0, /*FIOperandNum=*/1);
1579 }
1580
1581 MRI.replaceRegWith(FromReg: ScratchReg, ToReg: TmpGPR);
1582 MRI.clearVirtRegs();
1583}
1584
1585bool RISCVInstrInfo::reverseBranchCondition(
1586 SmallVectorImpl<MachineOperand> &Cond) const {
1587 assert((Cond.size() == 3) && "Invalid branch condition!");
1588
1589 Cond[0].setImm(RISCVCC::getInverseBranchOpcode(BCC: Cond[0].getImm()));
1590
1591 return false;
1592}
1593
1594// Return true if the instruction is a load immediate instruction (i.e.
1595// (ADDI x0, imm) or (BSETI x0, imm)).
1596static bool isLoadImm(const MachineInstr *MI, int64_t &Imm) {
1597 if (MI->getOpcode() == RISCV::ADDI && MI->getOperand(i: 1).isReg() &&
1598 MI->getOperand(i: 1).getReg() == RISCV::X0) {
1599 Imm = MI->getOperand(i: 2).getImm();
1600 return true;
1601 }
1602 // BSETI can be used to create power of 2 constants. Only 2048 is currently
1603 // interesting because it is 1 more than the maximum ADDI constant.
1604 if (MI->getOpcode() == RISCV::BSETI && MI->getOperand(i: 1).isReg() &&
1605 MI->getOperand(i: 1).getReg() == RISCV::X0 &&
1606 MI->getOperand(i: 2).getImm() == 11) {
1607 Imm = 2048;
1608 return true;
1609 }
1610 return false;
1611}
1612
1613bool RISCVInstrInfo::isFromLoadImm(const MachineRegisterInfo &MRI,
1614 const MachineOperand &Op, int64_t &Imm) {
1615 // Either a load from immediate instruction or X0.
1616 if (!Op.isReg())
1617 return false;
1618
1619 Register Reg = Op.getReg();
1620 if (Reg == RISCV::X0) {
1621 Imm = 0;
1622 return true;
1623 }
1624
1625 if (!Reg.isVirtual())
1626 return false;
1627
1628 const MachineInstr *DefMI = MRI.getVRegDef(Reg);
1629 return DefMI && isLoadImm(MI: DefMI, Imm);
1630}
1631
1632bool RISCVInstrInfo::optimizeCondBranch(MachineInstr &MI) const {
1633 bool IsSigned = false;
1634 bool IsEquality = false;
1635 switch (MI.getOpcode()) {
1636 default:
1637 return false;
1638 case RISCV::BEQ:
1639 case RISCV::BNE:
1640 IsEquality = true;
1641 break;
1642 case RISCV::BGE:
1643 case RISCV::BLT:
1644 IsSigned = true;
1645 break;
1646 case RISCV::BGEU:
1647 case RISCV::BLTU:
1648 break;
1649 }
1650
1651 MachineBasicBlock *MBB = MI.getParent();
1652 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
1653
1654 const MachineOperand &LHS = MI.getOperand(i: 0);
1655 const MachineOperand &RHS = MI.getOperand(i: 1);
1656 MachineBasicBlock *TBB = MI.getOperand(i: 2).getMBB();
1657
1658 RISCVCC::CondCode CC = getCondFromBranchOpc(Opc: MI.getOpcode());
1659 assert(CC != RISCVCC::COND_INVALID);
1660
1661 // Canonicalize conditional branches which can be constant folded into
1662 // beqz or bnez. We can't modify the CFG here.
1663 int64_t C0, C1;
1664 if (isFromLoadImm(MRI, Op: LHS, Imm&: C0) && isFromLoadImm(MRI, Op: RHS, Imm&: C1)) {
1665 unsigned NewOpc = evaluateCondBranch(CC, C0, C1) ? RISCV::BEQ : RISCV::BNE;
1666 // Build the new branch and remove the old one.
1667 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
1668 .addReg(RegNo: RISCV::X0)
1669 .addReg(RegNo: RISCV::X0)
1670 .addMBB(MBB: TBB);
1671 MI.eraseFromParent();
1672 return true;
1673 }
1674
1675 if (IsEquality)
1676 return false;
1677
1678 // For two constants C0 and C1 from
1679 // ```
1680 // li Y, C0
1681 // li Z, C1
1682 // ```
1683 // 1. if C1 = C0 + 1
1684 // we can turn:
1685 // (a) blt Y, X -> bge X, Z
1686 // (b) bge Y, X -> blt X, Z
1687 //
1688 // 2. if C1 = C0 - 1
1689 // we can turn:
1690 // (a) blt X, Y -> bge Z, X
1691 // (b) bge X, Y -> blt Z, X
1692 //
1693 // To make sure this optimization is really beneficial, we only
1694 // optimize for cases where Y had only one use (i.e. only used by the branch).
1695 // Try to find the register for constant Z; return
1696 // invalid register otherwise.
1697 auto searchConst = [&](int64_t C1) -> Register {
1698 MachineBasicBlock::reverse_iterator II(&MI), E = MBB->rend();
1699 auto DefC1 = std::find_if(first: ++II, last: E, pred: [&](const MachineInstr &I) -> bool {
1700 int64_t Imm;
1701 return isLoadImm(MI: &I, Imm) && Imm == C1 &&
1702 I.getOperand(i: 0).getReg().isVirtual();
1703 });
1704 if (DefC1 != E)
1705 return DefC1->getOperand(i: 0).getReg();
1706
1707 return Register();
1708 };
1709
1710 unsigned NewOpc = RISCVCC::getBrCond(CC: getInverseBranchCondition(CC));
1711
1712 // Might be case 1.
1713 // Don't change 0 to 1 since we can use x0.
1714 // For unsigned cases changing -1U to 0 would be incorrect.
1715 // The incorrect case for signed would be INT_MAX, but isFromLoadImm can't
1716 // return that.
1717 if (isFromLoadImm(MRI, Op: LHS, Imm&: C0) && C0 != 0 && LHS.getReg().isVirtual() &&
1718 MRI.hasOneUse(RegNo: LHS.getReg()) && (IsSigned || C0 != -1)) {
1719 assert((isInt<12>(C0) || C0 == 2048) && "Unexpected immediate");
1720 if (Register RegZ = searchConst(C0 + 1)) {
1721 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
1722 .add(MO: RHS)
1723 .addReg(RegNo: RegZ)
1724 .addMBB(MBB: TBB);
1725 MI.eraseFromParent();
1726 return true;
1727 }
1728 }
1729
1730 // Might be case 2.
1731 // For signed cases we don't want to change 0 since we can use x0.
1732 // For unsigned cases changing 0 to -1U would be incorrect.
1733 // The incorrect case for signed would be INT_MIN, but isFromLoadImm can't
1734 // return that.
1735 if (isFromLoadImm(MRI, Op: RHS, Imm&: C0) && C0 != 0 && RHS.getReg().isVirtual() &&
1736 MRI.hasOneUse(RegNo: RHS.getReg())) {
1737 assert((isInt<12>(C0) || C0 == 2048) && "Unexpected immediate");
1738 if (Register RegZ = searchConst(C0 - 1)) {
1739 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
1740 .addReg(RegNo: RegZ)
1741 .add(MO: LHS)
1742 .addMBB(MBB: TBB);
1743 MI.eraseFromParent();
1744 return true;
1745 }
1746 }
1747
1748 return false;
1749}
1750
1751MachineBasicBlock *
1752RISCVInstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
1753 assert(MI.getDesc().isBranch() && "Unexpected opcode!");
1754 // The branch target is always the last operand.
1755 int NumOp = MI.getNumExplicitOperands();
1756 return MI.getOperand(i: NumOp - 1).getMBB();
1757}
1758
1759bool RISCVInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
1760 int64_t BrOffset) const {
1761 unsigned XLen = STI.getXLen();
1762 // Ideally we could determine the supported branch offset from the
1763 // RISCVII::FormMask, but this can't be used for Pseudo instructions like
1764 // PseudoBR.
1765 switch (BranchOp) {
1766 default:
1767 llvm_unreachable("Unexpected opcode!");
1768 case RISCV::NDS_BBC:
1769 case RISCV::NDS_BBS:
1770 case RISCV::NDS_BEQC:
1771 case RISCV::NDS_BNEC:
1772 return isInt<11>(x: BrOffset);
1773 case RISCV::BEQ:
1774 case RISCV::BNE:
1775 case RISCV::BLT:
1776 case RISCV::BGE:
1777 case RISCV::BLTU:
1778 case RISCV::BGEU:
1779 case RISCV::BEQI:
1780 case RISCV::BNEI:
1781 case RISCV::CV_BEQIMM:
1782 case RISCV::CV_BNEIMM:
1783 case RISCV::QC_BEQI:
1784 case RISCV::QC_BNEI:
1785 case RISCV::QC_BGEI:
1786 case RISCV::QC_BLTI:
1787 case RISCV::QC_BLTUI:
1788 case RISCV::QC_BGEUI:
1789 case RISCV::QC_E_BEQI:
1790 case RISCV::QC_E_BNEI:
1791 case RISCV::QC_E_BGEI:
1792 case RISCV::QC_E_BLTI:
1793 case RISCV::QC_E_BLTUI:
1794 case RISCV::QC_E_BGEUI:
1795 return isInt<13>(x: BrOffset);
1796 case RISCV::JAL:
1797 case RISCV::PseudoBR:
1798 return isInt<21>(x: BrOffset);
1799 case RISCV::PseudoJump:
1800 return isInt<32>(x: SignExtend64(X: BrOffset + 0x800, B: XLen));
1801 }
1802}
1803
1804static bool isJumpTableLoad(const MachineInstr &MI) {
1805 return any_of(Range: MI.memoperands(), P: [](const MachineMemOperand *MMO) {
1806 const PseudoSourceValue *PSV = MMO->getPseudoValue();
1807 return PSV && PSV->isJumpTable();
1808 });
1809}
1810
1811// We want this instruction to be loading the base address of a jump table into
1812// a register. This can be PseudoMovAddr/PseudoLLA/LUI(+ADDI)/QC_E_LI.
1813static int getJumpTableIndexFromBase(const MachineRegisterInfo &MRI,
1814 Register Reg) {
1815 if (!Reg.isVirtual())
1816 return -1;
1817 const MachineInstr *MI = MRI.getUniqueVRegDef(Reg);
1818 if (!MI)
1819 return -1;
1820
1821 for (const MachineOperand &MO : MI->operands())
1822 if (MO.isJTI())
1823 return MO.getIndex();
1824
1825 return -1;
1826}
1827
1828// This instruction is used as the base address of a jump table load. We expect
1829// it to be adding the jump table base address to an index that may be scaled.
1830static int getJumpTableIndexFromLoadAddr(const MachineRegisterInfo &MRI,
1831 Register Reg) {
1832 if (!Reg.isVirtual())
1833 return -1;
1834 const MachineInstr *MI = MRI.getUniqueVRegDef(Reg);
1835 if (!MI)
1836 return -1;
1837
1838 int JTI;
1839 switch (MI->getOpcode()) {
1840 case RISCV::SH1ADD:
1841 case RISCV::SH2ADD:
1842 case RISCV::SH3ADD:
1843 // Only the index should be scaled so we just check the unscaled operand for
1844 // the base address.
1845 // TODO: Can the address be SHXADD_UW?
1846 JTI = getJumpTableIndexFromBase(MRI, Reg: MI->getOperand(i: 2).getReg());
1847 if (JTI >= 0)
1848 return JTI;
1849 break;
1850 case RISCV::ADD:
1851 JTI = getJumpTableIndexFromBase(MRI, Reg: MI->getOperand(i: 1).getReg());
1852 if (JTI >= 0)
1853 return JTI;
1854 JTI = getJumpTableIndexFromBase(MRI, Reg: MI->getOperand(i: 2).getReg());
1855 if (JTI >= 0)
1856 return JTI;
1857 break;
1858 }
1859
1860 return -1;
1861}
1862
1863// Recursively search for %jump-table.N starting from PseudoBRIND,
1864// and return the index of %jump-table.N.
1865//
1866// One common jump table:
1867//
1868// %base = PseudoMovAddr/PseudoLLA/LUI(+ADDI)/QC_E_LI %jump-table.N
1869// %addr = SH2ADD %index, %base
1870// %entry = LW %addr, 0 :: (load from jump-table)
1871// %target = ADD %entry, %base
1872// PseudoBRIND %target, 0
1873//
1874int RISCVInstrInfo::getJumpTableIndex(const MachineInstr &MI) const {
1875 if (MI.getOpcode() != RISCV::PseudoBRIND &&
1876 MI.getOpcode() != RISCV::PseudoBRINDX7)
1877 return -1;
1878
1879 Register Reg = MI.getOperand(i: 0).getReg();
1880 if (!Reg.isVirtual())
1881 return -1;
1882
1883 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1884 MachineInstr *Def = MRI.getUniqueVRegDef(Reg);
1885 if (!Def)
1886 return -1;
1887
1888 // The target may come directly from a load or the jump table may store
1889 // relative offset that needs the table base added to it.
1890 int JTI;
1891 switch (Def->getOpcode()) {
1892 case RISCV::LW:
1893 case RISCV::LWU:
1894 case RISCV::LD:
1895 if (!isJumpTableLoad(MI: *Def))
1896 return -1;
1897
1898 JTI = getJumpTableIndexFromLoadAddr(MRI, Reg: Def->getOperand(i: 1).getReg());
1899 if (JTI >= 0)
1900 return JTI;
1901 break;
1902 case RISCV::LXSW:
1903 case RISCV::LXWU:
1904 case RISCV::LXSD:
1905 case RISCV::QC_LRW:
1906 if (!isJumpTableLoad(MI: *Def))
1907 return -1;
1908
1909 JTI = getJumpTableIndexFromBase(MRI, Reg: Def->getOperand(i: 1).getReg());
1910 if (JTI >= 0)
1911 return JTI;
1912 break;
1913 case RISCV::ADD:
1914 JTI = getJumpTableIndexFromBase(MRI, Reg: Def->getOperand(i: 1).getReg());
1915 if (JTI >= 0)
1916 return JTI;
1917 JTI = getJumpTableIndexFromBase(MRI, Reg: Def->getOperand(i: 2).getReg());
1918 if (JTI >= 0)
1919 return JTI;
1920 break;
1921 }
1922
1923 return -1;
1924}
1925
1926// If the operation has a predicated pseudo instruction, return the pseudo
1927// instruction opcode. Otherwise, return RISCV::INSTRUCTION_LIST_END.
1928// TODO: Support more operations.
1929unsigned getPredicatedOpcode(unsigned Opcode) {
1930 // clang-format off
1931 switch (Opcode) {
1932 case RISCV::ADD: return RISCV::PseudoCCADD;
1933 case RISCV::SUB: return RISCV::PseudoCCSUB;
1934 case RISCV::SLL: return RISCV::PseudoCCSLL;
1935 case RISCV::SRL: return RISCV::PseudoCCSRL;
1936 case RISCV::SRA: return RISCV::PseudoCCSRA;
1937 case RISCV::AND: return RISCV::PseudoCCAND;
1938 case RISCV::OR: return RISCV::PseudoCCOR;
1939 case RISCV::XOR: return RISCV::PseudoCCXOR;
1940 case RISCV::MAX: return RISCV::PseudoCCMAX;
1941 case RISCV::MAXU: return RISCV::PseudoCCMAXU;
1942 case RISCV::MIN: return RISCV::PseudoCCMIN;
1943 case RISCV::MINU: return RISCV::PseudoCCMINU;
1944 case RISCV::MUL: return RISCV::PseudoCCMUL;
1945 case RISCV::LUI: return RISCV::PseudoCCLUI;
1946 case RISCV::QC_LI: return RISCV::PseudoCCQC_LI;
1947 case RISCV::QC_E_LI: return RISCV::PseudoCCQC_E_LI;
1948
1949 case RISCV::ADDI: return RISCV::PseudoCCADDI;
1950 case RISCV::SLLI: return RISCV::PseudoCCSLLI;
1951 case RISCV::SRLI: return RISCV::PseudoCCSRLI;
1952 case RISCV::SRAI: return RISCV::PseudoCCSRAI;
1953 case RISCV::ANDI: return RISCV::PseudoCCANDI;
1954 case RISCV::ORI: return RISCV::PseudoCCORI;
1955 case RISCV::XORI: return RISCV::PseudoCCXORI;
1956
1957 case RISCV::ADDW: return RISCV::PseudoCCADDW;
1958 case RISCV::SUBW: return RISCV::PseudoCCSUBW;
1959 case RISCV::SLLW: return RISCV::PseudoCCSLLW;
1960 case RISCV::SRLW: return RISCV::PseudoCCSRLW;
1961 case RISCV::SRAW: return RISCV::PseudoCCSRAW;
1962
1963 case RISCV::ADDIW: return RISCV::PseudoCCADDIW;
1964 case RISCV::SLLIW: return RISCV::PseudoCCSLLIW;
1965 case RISCV::SRLIW: return RISCV::PseudoCCSRLIW;
1966 case RISCV::SRAIW: return RISCV::PseudoCCSRAIW;
1967
1968 case RISCV::ANDN: return RISCV::PseudoCCANDN;
1969 case RISCV::ORN: return RISCV::PseudoCCORN;
1970 case RISCV::XNOR: return RISCV::PseudoCCXNOR;
1971
1972 case RISCV::NDS_BFOS: return RISCV::PseudoCCNDS_BFOS;
1973 case RISCV::NDS_BFOZ: return RISCV::PseudoCCNDS_BFOZ;
1974 }
1975 // clang-format on
1976
1977 return RISCV::INSTRUCTION_LIST_END;
1978}
1979
1980/// Identify instructions that can be folded into a CCMOV instruction, and
1981/// return the defining instruction.
1982static MachineInstr *canFoldAsPredicatedOp(Register Reg,
1983 const MachineRegisterInfo &MRI,
1984 const TargetInstrInfo *TII,
1985 const RISCVSubtarget &STI) {
1986 if (!Reg.isVirtual())
1987 return nullptr;
1988 if (!MRI.hasOneNonDBGUse(RegNo: Reg))
1989 return nullptr;
1990 MachineInstr *MI = MRI.getVRegDef(Reg);
1991 if (!MI)
1992 return nullptr;
1993
1994 if (!STI.hasShortForwardBranchIMinMax() &&
1995 (MI->getOpcode() == RISCV::MAX || MI->getOpcode() == RISCV::MIN ||
1996 MI->getOpcode() == RISCV::MINU || MI->getOpcode() == RISCV::MAXU))
1997 return nullptr;
1998
1999 if (!STI.hasShortForwardBranchIMul() && MI->getOpcode() == RISCV::MUL)
2000 return nullptr;
2001
2002 // Check if MI can be predicated and folded into the CCMOV.
2003 if (getPredicatedOpcode(Opcode: MI->getOpcode()) == RISCV::INSTRUCTION_LIST_END)
2004 return nullptr;
2005 // Don't predicate li idiom.
2006 if (MI->getOpcode() == RISCV::ADDI && MI->getOperand(i: 1).isReg() &&
2007 MI->getOperand(i: 1).getReg() == RISCV::X0)
2008 return nullptr;
2009 // Check if MI has any other defs or physreg uses.
2010 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands())) {
2011 // Reject frame index operands, PEI can't handle the predicated pseudos.
2012 if (MO.isFI() || MO.isCPI() || MO.isJTI())
2013 return nullptr;
2014 if (!MO.isReg())
2015 continue;
2016 // MI can't have any tied operands, that would conflict with predication.
2017 if (MO.isTied())
2018 return nullptr;
2019 if (MO.isDef())
2020 return nullptr;
2021 // Allow constant physregs.
2022 if (MO.getReg().isPhysical() && !MRI.isConstantPhysReg(PhysReg: MO.getReg()))
2023 return nullptr;
2024 }
2025 bool DontMoveAcrossStores = true;
2026 if (!MI->isSafeToMove(SawStore&: DontMoveAcrossStores))
2027 return nullptr;
2028 return MI;
2029}
2030
2031MachineInstr *
2032RISCVInstrInfo::optimizeSelect(MachineInstr &MI,
2033 SmallPtrSetImpl<MachineInstr *> &SeenMIs,
2034 bool PreferFalse) const {
2035 assert(MI.getOpcode() == RISCV::PseudoCCMOVGPR &&
2036 "Unknown select instruction");
2037 if (!STI.hasShortForwardBranchIALU())
2038 return nullptr;
2039
2040 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
2041 MachineInstr *DefMI =
2042 canFoldAsPredicatedOp(Reg: MI.getOperand(i: 2).getReg(), MRI, TII: this, STI);
2043 bool Invert = !DefMI;
2044 if (!DefMI)
2045 DefMI = canFoldAsPredicatedOp(Reg: MI.getOperand(i: 1).getReg(), MRI, TII: this, STI);
2046 if (!DefMI)
2047 return nullptr;
2048
2049 // Find new register class to use.
2050 MachineOperand FalseReg = MI.getOperand(i: Invert ? 2 : 1);
2051 Register DestReg = MI.getOperand(i: 0).getReg();
2052 const TargetRegisterClass *PreviousClass = MRI.getRegClass(Reg: FalseReg.getReg());
2053 if (!MRI.constrainRegClass(Reg: DestReg, RC: PreviousClass))
2054 return nullptr;
2055
2056 unsigned PredOpc = getPredicatedOpcode(Opcode: DefMI->getOpcode());
2057 assert(PredOpc != RISCV::INSTRUCTION_LIST_END && "Unexpected opcode!");
2058
2059 // Create a new predicated version of DefMI.
2060 MachineInstrBuilder NewMI =
2061 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: PredOpc), DestReg);
2062
2063 // Copy the false register.
2064 NewMI.add(MO: FalseReg);
2065
2066 // Copy all the DefMI operands.
2067 const MCInstrDesc &DefDesc = DefMI->getDesc();
2068 for (unsigned i = 1, e = DefDesc.getNumOperands(); i != e; ++i)
2069 NewMI.add(MO: DefMI->getOperand(i));
2070
2071 // Add branch opcode, inverting if necessary.
2072 unsigned BCCOpcode = MI.getOperand(i: MI.getNumExplicitOperands() - 3).getImm();
2073 if (Invert)
2074 BCCOpcode = RISCVCC::getInverseBranchOpcode(BCC: BCCOpcode);
2075 NewMI.addImm(Val: BCCOpcode);
2076
2077 // Copy the condition portion.
2078 NewMI.add(MO: MI.getOperand(i: MI.getNumExplicitOperands() - 2));
2079 NewMI.add(MO: MI.getOperand(i: MI.getNumExplicitOperands() - 1));
2080
2081 // Update SeenMIs set: register newly created MI and erase removed DefMI.
2082 SeenMIs.insert(Ptr: NewMI);
2083 SeenMIs.erase(Ptr: DefMI);
2084
2085 // If MI is inside a loop, and DefMI is outside the loop, then kill flags on
2086 // DefMI would be invalid when transferred inside the loop. Checking for a
2087 // loop is expensive, but at least remove kill flags if they are in different
2088 // BBs.
2089 if (DefMI->getParent() != MI.getParent())
2090 NewMI->clearKillInfo();
2091
2092 // The caller will erase MI, but not DefMI.
2093 DefMI->eraseFromParent();
2094 return NewMI;
2095}
2096
2097unsigned RISCVInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
2098 if (MI.isMetaInstruction())
2099 return 0;
2100
2101 unsigned Opcode = MI.getOpcode();
2102
2103 if (Opcode == TargetOpcode::INLINEASM ||
2104 Opcode == TargetOpcode::INLINEASM_BR) {
2105 const MachineFunction &MF = *MI.getParent()->getParent();
2106 return getInlineAsmLength(Str: MI.getOperand(i: 0).getSymbolName(),
2107 MAI: MF.getTarget().getMCAsmInfo());
2108 }
2109
2110 if (requiresNTLHint(MI)) {
2111 if (STI.hasStdExtZca()) {
2112 if (unsigned Size = getCompressedSize(MI, STI))
2113 return 2 + Size; // c.ntl.all + c.load/c.store
2114 return 6; // c.ntl.all + load/store
2115 }
2116 return 8; // ntl.all + load/store
2117 }
2118
2119 if (Opcode == TargetOpcode::BUNDLE)
2120 return getInstBundleSize(MI);
2121
2122 if (MI.getParent() && MI.getParent()->getParent()) {
2123 if (unsigned Size = getCompressedSize(MI, STI))
2124 return Size;
2125 }
2126
2127 switch (Opcode) {
2128 case RISCV::PseudoMV_FPR16INX:
2129 case RISCV::PseudoMV_FPR32INX:
2130 case RISCV::PseudoClearGPR:
2131 // MV is always compressible to either c.mv or c.li rd, 0.
2132 return STI.hasStdExtZca() ? 2 : 4;
2133 // Below cases are for short forward branch pseudos
2134 case RISCV::PseudoCCMOVGPRNoX0:
2135 return get(Opcode: MI.getOperand(i: MI.getNumExplicitOperands() - 3).getImm())
2136 .getSize() +
2137 2;
2138 case RISCV::PseudoCCMOVGPR:
2139 case RISCV::PseudoCCADD:
2140 case RISCV::PseudoCCSUB:
2141 case RISCV::PseudoCCSLL:
2142 case RISCV::PseudoCCSRL:
2143 case RISCV::PseudoCCSRA:
2144 case RISCV::PseudoCCAND:
2145 case RISCV::PseudoCCOR:
2146 case RISCV::PseudoCCXOR:
2147 case RISCV::PseudoCCADDI:
2148 case RISCV::PseudoCCANDI:
2149 case RISCV::PseudoCCORI:
2150 case RISCV::PseudoCCXORI:
2151 case RISCV::PseudoCCLUI:
2152 case RISCV::PseudoCCSLLI:
2153 case RISCV::PseudoCCSRLI:
2154 case RISCV::PseudoCCSRAI:
2155 case RISCV::PseudoCCADDW:
2156 case RISCV::PseudoCCSUBW:
2157 case RISCV::PseudoCCSLLW:
2158 case RISCV::PseudoCCSRLW:
2159 case RISCV::PseudoCCSRAW:
2160 case RISCV::PseudoCCADDIW:
2161 case RISCV::PseudoCCSLLIW:
2162 case RISCV::PseudoCCSRLIW:
2163 case RISCV::PseudoCCSRAIW:
2164 case RISCV::PseudoCCANDN:
2165 case RISCV::PseudoCCORN:
2166 case RISCV::PseudoCCXNOR:
2167 case RISCV::PseudoCCMAX:
2168 case RISCV::PseudoCCMIN:
2169 case RISCV::PseudoCCMAXU:
2170 case RISCV::PseudoCCMINU:
2171 case RISCV::PseudoCCMUL:
2172 case RISCV::PseudoCCLB:
2173 case RISCV::PseudoCCLH:
2174 case RISCV::PseudoCCLW:
2175 case RISCV::PseudoCCLHU:
2176 case RISCV::PseudoCCLBU:
2177 case RISCV::PseudoCCLWU:
2178 case RISCV::PseudoCCLD:
2179 case RISCV::PseudoCCQC_LI:
2180 return get(Opcode: MI.getOperand(i: MI.getNumExplicitOperands() - 3).getImm())
2181 .getSize() +
2182 4;
2183 case RISCV::PseudoCCQC_E_LI:
2184 case RISCV::PseudoCCQC_E_LB:
2185 case RISCV::PseudoCCQC_E_LH:
2186 case RISCV::PseudoCCQC_E_LW:
2187 case RISCV::PseudoCCQC_E_LHU:
2188 case RISCV::PseudoCCQC_E_LBU:
2189 return get(Opcode: MI.getOperand(i: MI.getNumExplicitOperands() - 3).getImm())
2190 .getSize() +
2191 6;
2192 case TargetOpcode::STACKMAP:
2193 // The upper bound for a stackmap intrinsic is the full length of its shadow
2194 return StackMapOpers(&MI).getNumPatchBytes();
2195 case TargetOpcode::PATCHPOINT:
2196 // The size of the patchpoint intrinsic is the number of bytes requested
2197 return PatchPointOpers(&MI).getNumPatchBytes();
2198 case TargetOpcode::STATEPOINT: {
2199 // The size of the statepoint intrinsic is the number of bytes requested
2200 unsigned NumBytes = StatepointOpers(&MI).getNumPatchBytes();
2201 // No patch bytes means at most a PseudoCall is emitted
2202 return std::max(a: NumBytes, b: 8U);
2203 }
2204 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
2205 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
2206 case TargetOpcode::PATCHABLE_TAIL_CALL: {
2207 const MachineFunction &MF = *MI.getParent()->getParent();
2208 const Function &F = MF.getFunction();
2209 if (Opcode == TargetOpcode::PATCHABLE_FUNCTION_ENTER &&
2210 F.hasFnAttribute(Kind: "patchable-function-entry")) {
2211 unsigned Num =
2212 F.getFnAttributeAsParsedInteger(Kind: "patchable-function-entry");
2213 // Number of C.NOP or NOP
2214 return (STI.hasStdExtZca() ? 2 : 4) * Num;
2215 }
2216 // XRay uses C.JAL + 21 or 33 C.NOP for each sled in RV32 and RV64,
2217 // respectively.
2218 return STI.is64Bit() ? 68 : 44;
2219 }
2220 default:
2221 return get(Opcode).getSize();
2222 }
2223}
2224
2225bool RISCVInstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const {
2226 const unsigned Opcode = MI.getOpcode();
2227 switch (Opcode) {
2228 default:
2229 break;
2230 case RISCV::FSGNJ_D:
2231 case RISCV::FSGNJ_S:
2232 case RISCV::FSGNJ_H:
2233 case RISCV::FSGNJ_D_INX:
2234 case RISCV::FSGNJ_D_IN32X:
2235 case RISCV::FSGNJ_S_INX:
2236 case RISCV::FSGNJ_H_INX:
2237 // The canonical floating-point move is fsgnj rd, rs, rs.
2238 return MI.getOperand(i: 1).isReg() && MI.getOperand(i: 2).isReg() &&
2239 MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg();
2240 case RISCV::ADDI:
2241 case RISCV::ORI:
2242 case RISCV::XORI:
2243 return (MI.getOperand(i: 1).isReg() &&
2244 MI.getOperand(i: 1).getReg() == RISCV::X0) ||
2245 (MI.getOperand(i: 2).isImm() && MI.getOperand(i: 2).getImm() == 0);
2246 }
2247 return MI.isAsCheapAsAMove();
2248}
2249
2250std::optional<DestSourcePair>
2251RISCVInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
2252 if (MI.isMoveReg())
2253 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2254 switch (MI.getOpcode()) {
2255 default:
2256 break;
2257 case RISCV::ADD:
2258 case RISCV::OR:
2259 case RISCV::XOR:
2260 if (MI.getOperand(i: 1).isReg() && MI.getOperand(i: 1).getReg() == RISCV::X0 &&
2261 MI.getOperand(i: 2).isReg())
2262 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 2)};
2263 if (MI.getOperand(i: 2).isReg() && MI.getOperand(i: 2).getReg() == RISCV::X0 &&
2264 MI.getOperand(i: 1).isReg())
2265 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2266 break;
2267 case RISCV::ADDI:
2268 // Operand 1 can be a frameindex but callers expect registers
2269 if (MI.getOperand(i: 1).isReg() && MI.getOperand(i: 2).isImm() &&
2270 MI.getOperand(i: 2).getImm() == 0)
2271 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2272 break;
2273 case RISCV::SUB:
2274 if (MI.getOperand(i: 2).isReg() && MI.getOperand(i: 2).getReg() == RISCV::X0 &&
2275 MI.getOperand(i: 1).isReg())
2276 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2277 break;
2278 case RISCV::SH1ADD:
2279 case RISCV::SH1ADD_UW:
2280 case RISCV::SH2ADD:
2281 case RISCV::SH2ADD_UW:
2282 case RISCV::SH3ADD:
2283 case RISCV::SH3ADD_UW:
2284 if (MI.getOperand(i: 1).isReg() && MI.getOperand(i: 1).getReg() == RISCV::X0 &&
2285 MI.getOperand(i: 2).isReg())
2286 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 2)};
2287 break;
2288 case RISCV::FSGNJ_D:
2289 case RISCV::FSGNJ_S:
2290 case RISCV::FSGNJ_H:
2291 case RISCV::FSGNJ_D_INX:
2292 case RISCV::FSGNJ_D_IN32X:
2293 case RISCV::FSGNJ_S_INX:
2294 case RISCV::FSGNJ_H_INX:
2295 // The canonical floating-point move is fsgnj rd, rs, rs.
2296 if (MI.getOperand(i: 1).isReg() && MI.getOperand(i: 2).isReg() &&
2297 MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg())
2298 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2299 break;
2300 }
2301 return std::nullopt;
2302}
2303
2304MachineTraceStrategy RISCVInstrInfo::getMachineCombinerTraceStrategy() const {
2305 std::optional<MachineTraceStrategy> Forced =
2306 STI.getCLOpts().force_machine_combiner_strategy;
2307 if (!Forced) {
2308 // The option is unused. Choose Local strategy only for in-order cores. When
2309 // scheduling model is unspecified, use MinInstrCount strategy as more
2310 // generic one.
2311 const auto &SchedModel = STI.getSchedModel();
2312 return (!SchedModel.hasInstrSchedModel() || SchedModel.isOutOfOrder())
2313 ? MachineTraceStrategy::TS_MinInstrCount
2314 : MachineTraceStrategy::TS_Local;
2315 }
2316 // The strategy was forced by the option.
2317 return *Forced;
2318}
2319
2320void RISCVInstrInfo::finalizeInsInstrs(
2321 MachineInstr &Root, unsigned &Pattern,
2322 SmallVectorImpl<MachineInstr *> &InsInstrs) const {
2323 int16_t FrmOpIdx =
2324 RISCV::getNamedOperandIdx(Opcode: Root.getOpcode(), Name: RISCV::OpName::frm);
2325 if (FrmOpIdx < 0) {
2326 assert(all_of(InsInstrs,
2327 [](MachineInstr *MI) {
2328 return RISCV::getNamedOperandIdx(MI->getOpcode(),
2329 RISCV::OpName::frm) < 0;
2330 }) &&
2331 "New instructions require FRM whereas the old one does not have it");
2332 return;
2333 }
2334
2335 const MachineOperand &FRM = Root.getOperand(i: FrmOpIdx);
2336 MachineFunction &MF = *Root.getMF();
2337
2338 for (auto *NewMI : InsInstrs) {
2339 // We'd already added the FRM operand.
2340 if (static_cast<unsigned>(RISCV::getNamedOperandIdx(
2341 Opcode: NewMI->getOpcode(), Name: RISCV::OpName::frm)) != NewMI->getNumOperands())
2342 continue;
2343 MachineInstrBuilder MIB(MF, NewMI);
2344 MIB.add(MO: FRM);
2345 if (FRM.getImm() == RISCVFPRndMode::DYN)
2346 MIB.addUse(RegNo: RISCV::FRM, Flags: RegState::Implicit);
2347 }
2348}
2349
2350static bool isFADD(unsigned Opc) {
2351 switch (Opc) {
2352 default:
2353 return false;
2354 case RISCV::FADD_H:
2355 case RISCV::FADD_S:
2356 case RISCV::FADD_D:
2357 return true;
2358 }
2359}
2360
2361static bool isFSUB(unsigned Opc) {
2362 switch (Opc) {
2363 default:
2364 return false;
2365 case RISCV::FSUB_H:
2366 case RISCV::FSUB_S:
2367 case RISCV::FSUB_D:
2368 return true;
2369 }
2370}
2371
2372static bool isFMUL(unsigned Opc) {
2373 switch (Opc) {
2374 default:
2375 return false;
2376 case RISCV::FMUL_H:
2377 case RISCV::FMUL_S:
2378 case RISCV::FMUL_D:
2379 return true;
2380 }
2381}
2382
2383bool RISCVInstrInfo::isVectorAssociativeAndCommutative(const MachineInstr &Inst,
2384 bool Invert) const {
2385#define OPCODE_LMUL_CASE(OPC) \
2386 case RISCV::OPC##_M1: \
2387 case RISCV::OPC##_M2: \
2388 case RISCV::OPC##_M4: \
2389 case RISCV::OPC##_M8: \
2390 case RISCV::OPC##_MF2: \
2391 case RISCV::OPC##_MF4: \
2392 case RISCV::OPC##_MF8
2393
2394#define OPCODE_LMUL_MASK_CASE(OPC) \
2395 case RISCV::OPC##_M1_MASK: \
2396 case RISCV::OPC##_M2_MASK: \
2397 case RISCV::OPC##_M4_MASK: \
2398 case RISCV::OPC##_M8_MASK: \
2399 case RISCV::OPC##_MF2_MASK: \
2400 case RISCV::OPC##_MF4_MASK: \
2401 case RISCV::OPC##_MF8_MASK
2402
2403 unsigned Opcode = Inst.getOpcode();
2404 if (Invert) {
2405 if (auto InvOpcode = getInverseOpcode(Opcode))
2406 Opcode = *InvOpcode;
2407 else
2408 return false;
2409 }
2410
2411 // clang-format off
2412 switch (Opcode) {
2413 default:
2414 return false;
2415 OPCODE_LMUL_CASE(PseudoVADD_VV):
2416 OPCODE_LMUL_MASK_CASE(PseudoVADD_VV):
2417 OPCODE_LMUL_CASE(PseudoVMUL_VV):
2418 OPCODE_LMUL_MASK_CASE(PseudoVMUL_VV):
2419 return true;
2420 }
2421 // clang-format on
2422
2423#undef OPCODE_LMUL_MASK_CASE
2424#undef OPCODE_LMUL_CASE
2425}
2426
2427bool RISCVInstrInfo::areRVVInstsReassociable(const MachineInstr &Root,
2428 const MachineInstr &Prev) const {
2429 if (!areOpcodesEqualOrInverse(Opcode1: Root.getOpcode(), Opcode2: Prev.getOpcode()))
2430 return false;
2431
2432 assert(Root.getMF() == Prev.getMF());
2433 const MachineRegisterInfo *MRI = &Root.getMF()->getRegInfo();
2434 const TargetRegisterInfo *TRI = MRI->getTargetRegisterInfo();
2435
2436 // Make sure vtype operands are also the same.
2437 const MCInstrDesc &Desc = get(Opcode: Root.getOpcode());
2438 const uint64_t TSFlags = Desc.TSFlags;
2439
2440 auto checkImmOperand = [&](unsigned OpIdx) {
2441 return Root.getOperand(i: OpIdx).getImm() == Prev.getOperand(i: OpIdx).getImm();
2442 };
2443
2444 auto checkRegOperand = [&](unsigned OpIdx) {
2445 return Root.getOperand(i: OpIdx).getReg() == Prev.getOperand(i: OpIdx).getReg();
2446 };
2447
2448 // PassThru
2449 // TODO: Potentially we can loosen the condition to consider Root to be
2450 // associable with Prev if Root has NoReg as passthru. In which case we
2451 // also need to loosen the condition on vector policies between these.
2452 if (!checkRegOperand(1))
2453 return false;
2454
2455 // SEW
2456 if (RISCVII::hasSEWOp(TSFlags) &&
2457 !checkImmOperand(RISCVII::getSEWOpNum(Desc)))
2458 return false;
2459
2460 // Mask
2461 if (RISCVII::usesMaskPolicy(TSFlags)) {
2462 const MachineBasicBlock *MBB = Root.getParent();
2463 const MachineBasicBlock::const_reverse_iterator It1(&Root);
2464 const MachineBasicBlock::const_reverse_iterator It2(&Prev);
2465 Register MI1VReg;
2466
2467 bool SeenMI2 = false;
2468 for (auto End = MBB->rend(), It = It1; It != End; ++It) {
2469 if (It == It2) {
2470 SeenMI2 = true;
2471 if (!MI1VReg.isValid())
2472 // There is no V0 def between Root and Prev; they're sharing the
2473 // same V0.
2474 break;
2475 }
2476
2477 if (It->modifiesRegister(Reg: RISCV::V0, TRI)) {
2478 Register SrcReg = It->getOperand(i: 1).getReg();
2479 // If it's not VReg it'll be more difficult to track its defs, so
2480 // bailing out here just to be safe.
2481 if (!SrcReg.isVirtual())
2482 return false;
2483
2484 if (!MI1VReg.isValid()) {
2485 // This is the V0 def for Root.
2486 MI1VReg = SrcReg;
2487 continue;
2488 }
2489
2490 // Some random mask updates.
2491 if (!SeenMI2)
2492 continue;
2493
2494 // This is the V0 def for Prev; check if it's the same as that of
2495 // Root.
2496 if (MI1VReg != SrcReg)
2497 return false;
2498 else
2499 break;
2500 }
2501 }
2502
2503 // If we haven't encountered Prev, it's likely that this function was
2504 // called in a wrong way (e.g. Root is before Prev).
2505 assert(SeenMI2 && "Prev is expected to appear before Root");
2506 }
2507
2508 // Tail / Mask policies
2509 if (RISCVII::hasVecPolicyOp(TSFlags) &&
2510 !checkImmOperand(RISCVII::getVecPolicyOpNum(Desc)))
2511 return false;
2512
2513 // VL
2514 if (RISCVII::hasVLOp(TSFlags)) {
2515 unsigned OpIdx = RISCVII::getVLOpNum(Desc);
2516 const MachineOperand &Op1 = Root.getOperand(i: OpIdx);
2517 const MachineOperand &Op2 = Prev.getOperand(i: OpIdx);
2518 if (Op1.getType() != Op2.getType())
2519 return false;
2520 switch (Op1.getType()) {
2521 case MachineOperand::MO_Register:
2522 if (Op1.getReg() != Op2.getReg())
2523 return false;
2524 break;
2525 case MachineOperand::MO_Immediate:
2526 if (Op1.getImm() != Op2.getImm())
2527 return false;
2528 break;
2529 default:
2530 llvm_unreachable("Unrecognized VL operand type");
2531 }
2532 }
2533
2534 // Rounding modes
2535 if (int Idx = RISCVII::getFRMOpNum(Desc); Idx >= 0 && !checkImmOperand(Idx))
2536 return false;
2537 if (int Idx = RISCVII::getVXRMOpNum(Desc); Idx >= 0 && !checkImmOperand(Idx))
2538 return false;
2539
2540 return true;
2541}
2542
2543// Most of our RVV pseudos have passthru operand, so the real operands
2544// start from index = 2.
2545bool RISCVInstrInfo::hasReassociableVectorSibling(const MachineInstr &Inst,
2546 bool &Commuted) const {
2547 const MachineBasicBlock *MBB = Inst.getParent();
2548 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
2549 assert(RISCVII::isFirstDefTiedToFirstUse(get(Inst.getOpcode())) &&
2550 "Expect the present of passthrough operand.");
2551 MachineInstr *MI1 = MRI.getUniqueVRegDef(Reg: Inst.getOperand(i: 2).getReg());
2552 MachineInstr *MI2 = MRI.getUniqueVRegDef(Reg: Inst.getOperand(i: 3).getReg());
2553
2554 // If only one operand has the same or inverse opcode and it's the second
2555 // source operand, the operands must be commuted.
2556 Commuted = !areRVVInstsReassociable(Root: Inst, Prev: *MI1) &&
2557 areRVVInstsReassociable(Root: Inst, Prev: *MI2);
2558 if (Commuted)
2559 std::swap(a&: MI1, b&: MI2);
2560
2561 return areRVVInstsReassociable(Root: Inst, Prev: *MI1) &&
2562 (isVectorAssociativeAndCommutative(Inst: *MI1) ||
2563 isVectorAssociativeAndCommutative(Inst: *MI1, /* Invert */ true)) &&
2564 hasReassociableOperands(Inst: *MI1, MBB) &&
2565 MRI.hasOneNonDBGUse(RegNo: MI1->getOperand(i: 0).getReg());
2566}
2567
2568bool RISCVInstrInfo::hasReassociableOperands(
2569 const MachineInstr &Inst, const MachineBasicBlock *MBB) const {
2570 if (!isVectorAssociativeAndCommutative(Inst) &&
2571 !isVectorAssociativeAndCommutative(Inst, /*Invert=*/true))
2572 return TargetInstrInfo::hasReassociableOperands(Inst, MBB);
2573
2574 const MachineOperand &Op1 = Inst.getOperand(i: 2);
2575 const MachineOperand &Op2 = Inst.getOperand(i: 3);
2576 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
2577
2578 // We need virtual register definitions for the operands that we will
2579 // reassociate.
2580 MachineInstr *MI1 = nullptr;
2581 MachineInstr *MI2 = nullptr;
2582 if (Op1.isReg() && Op1.getReg().isVirtual())
2583 MI1 = MRI.getUniqueVRegDef(Reg: Op1.getReg());
2584 if (Op2.isReg() && Op2.getReg().isVirtual())
2585 MI2 = MRI.getUniqueVRegDef(Reg: Op2.getReg());
2586
2587 // And at least one operand must be defined in MBB.
2588 return MI1 && MI2 && (MI1->getParent() == MBB || MI2->getParent() == MBB);
2589}
2590
2591void RISCVInstrInfo::getReassociateOperandIndices(
2592 const MachineInstr &Root, unsigned Pattern,
2593 std::array<unsigned, 5> &OperandIndices) const {
2594 TargetInstrInfo::getReassociateOperandIndices(Root, Pattern, OperandIndices);
2595 if (RISCV::getRVVMCOpcode(RVVPseudoOpcode: Root.getOpcode())) {
2596 // Skip the passthrough operand, so increment all indices by one.
2597 for (unsigned I = 0; I < 5; ++I)
2598 ++OperandIndices[I];
2599 }
2600}
2601
2602bool RISCVInstrInfo::hasReassociableSibling(const MachineInstr &Inst,
2603 bool &Commuted) const {
2604 if (isVectorAssociativeAndCommutative(Inst) ||
2605 isVectorAssociativeAndCommutative(Inst, /*Invert=*/true))
2606 return hasReassociableVectorSibling(Inst, Commuted);
2607
2608 if (!TargetInstrInfo::hasReassociableSibling(Inst, Commuted))
2609 return false;
2610
2611 const MachineRegisterInfo &MRI = Inst.getMF()->getRegInfo();
2612 unsigned OperandIdx = Commuted ? 2 : 1;
2613 const MachineInstr &Sibling =
2614 *MRI.getVRegDef(Reg: Inst.getOperand(i: OperandIdx).getReg());
2615
2616 int16_t InstFrmOpIdx =
2617 RISCV::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name: RISCV::OpName::frm);
2618 int16_t SiblingFrmOpIdx =
2619 RISCV::getNamedOperandIdx(Opcode: Sibling.getOpcode(), Name: RISCV::OpName::frm);
2620
2621 return (InstFrmOpIdx < 0 && SiblingFrmOpIdx < 0) ||
2622 RISCV::hasEqualFRM(MI1: Inst, MI2: Sibling);
2623}
2624
2625bool RISCVInstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst,
2626 bool Invert) const {
2627 if (isVectorAssociativeAndCommutative(Inst, Invert))
2628 return true;
2629
2630 unsigned Opc = Inst.getOpcode();
2631 if (Invert) {
2632 auto InverseOpcode = getInverseOpcode(Opcode: Opc);
2633 if (!InverseOpcode)
2634 return false;
2635 Opc = *InverseOpcode;
2636 }
2637
2638 if (isFADD(Opc) || isFMUL(Opc))
2639 return Inst.getFlag(Flag: MachineInstr::MIFlag::FmReassoc) &&
2640 Inst.getFlag(Flag: MachineInstr::MIFlag::FmNsz);
2641
2642 switch (Opc) {
2643 default:
2644 return false;
2645 case RISCV::ADD:
2646 case RISCV::ADDW:
2647 case RISCV::AND:
2648 case RISCV::OR:
2649 case RISCV::XOR:
2650 // From RISC-V ISA spec, if both the high and low bits of the same product
2651 // are required, then the recommended code sequence is:
2652 //
2653 // MULH[[S]U] rdh, rs1, rs2
2654 // MUL rdl, rs1, rs2
2655 // (source register specifiers must be in same order and rdh cannot be the
2656 // same as rs1 or rs2)
2657 //
2658 // Microarchitectures can then fuse these into a single multiply operation
2659 // instead of performing two separate multiplies.
2660 // MachineCombiner may reassociate MUL operands and lose the fusion
2661 // opportunity.
2662 case RISCV::MUL:
2663 case RISCV::MULW:
2664 case RISCV::MIN:
2665 case RISCV::MINU:
2666 case RISCV::MAX:
2667 case RISCV::MAXU:
2668 case RISCV::FMIN_H:
2669 case RISCV::FMIN_S:
2670 case RISCV::FMIN_D:
2671 case RISCV::FMAX_H:
2672 case RISCV::FMAX_S:
2673 case RISCV::FMAX_D:
2674 return true;
2675 }
2676
2677 return false;
2678}
2679
2680std::optional<unsigned>
2681RISCVInstrInfo::getInverseOpcode(unsigned Opcode) const {
2682#define RVV_OPC_LMUL_CASE(OPC, INV) \
2683 case RISCV::OPC##_M1: \
2684 return RISCV::INV##_M1; \
2685 case RISCV::OPC##_M2: \
2686 return RISCV::INV##_M2; \
2687 case RISCV::OPC##_M4: \
2688 return RISCV::INV##_M4; \
2689 case RISCV::OPC##_M8: \
2690 return RISCV::INV##_M8; \
2691 case RISCV::OPC##_MF2: \
2692 return RISCV::INV##_MF2; \
2693 case RISCV::OPC##_MF4: \
2694 return RISCV::INV##_MF4; \
2695 case RISCV::OPC##_MF8: \
2696 return RISCV::INV##_MF8
2697
2698#define RVV_OPC_LMUL_MASK_CASE(OPC, INV) \
2699 case RISCV::OPC##_M1_MASK: \
2700 return RISCV::INV##_M1_MASK; \
2701 case RISCV::OPC##_M2_MASK: \
2702 return RISCV::INV##_M2_MASK; \
2703 case RISCV::OPC##_M4_MASK: \
2704 return RISCV::INV##_M4_MASK; \
2705 case RISCV::OPC##_M8_MASK: \
2706 return RISCV::INV##_M8_MASK; \
2707 case RISCV::OPC##_MF2_MASK: \
2708 return RISCV::INV##_MF2_MASK; \
2709 case RISCV::OPC##_MF4_MASK: \
2710 return RISCV::INV##_MF4_MASK; \
2711 case RISCV::OPC##_MF8_MASK: \
2712 return RISCV::INV##_MF8_MASK
2713
2714 switch (Opcode) {
2715 default:
2716 return std::nullopt;
2717 case RISCV::FADD_H:
2718 return RISCV::FSUB_H;
2719 case RISCV::FADD_S:
2720 return RISCV::FSUB_S;
2721 case RISCV::FADD_D:
2722 return RISCV::FSUB_D;
2723 case RISCV::FSUB_H:
2724 return RISCV::FADD_H;
2725 case RISCV::FSUB_S:
2726 return RISCV::FADD_S;
2727 case RISCV::FSUB_D:
2728 return RISCV::FADD_D;
2729 case RISCV::ADD:
2730 return RISCV::SUB;
2731 case RISCV::SUB:
2732 return RISCV::ADD;
2733 case RISCV::ADDW:
2734 return RISCV::SUBW;
2735 case RISCV::SUBW:
2736 return RISCV::ADDW;
2737 // clang-format off
2738 RVV_OPC_LMUL_CASE(PseudoVADD_VV, PseudoVSUB_VV);
2739 RVV_OPC_LMUL_MASK_CASE(PseudoVADD_VV, PseudoVSUB_VV);
2740 RVV_OPC_LMUL_CASE(PseudoVSUB_VV, PseudoVADD_VV);
2741 RVV_OPC_LMUL_MASK_CASE(PseudoVSUB_VV, PseudoVADD_VV);
2742 // clang-format on
2743 }
2744
2745#undef RVV_OPC_LMUL_MASK_CASE
2746#undef RVV_OPC_LMUL_CASE
2747}
2748
2749static bool canCombineFPFusedMultiply(const MachineInstr &Root,
2750 const MachineOperand &MO,
2751 bool DoRegPressureReduce) {
2752 if (!MO.isReg() || !MO.getReg().isVirtual())
2753 return false;
2754 const MachineRegisterInfo &MRI = Root.getMF()->getRegInfo();
2755 MachineInstr *MI = MRI.getVRegDef(Reg: MO.getReg());
2756 if (!MI || !isFMUL(Opc: MI->getOpcode()))
2757 return false;
2758
2759 if (!Root.getFlag(Flag: MachineInstr::MIFlag::FmContract) ||
2760 !MI->getFlag(Flag: MachineInstr::MIFlag::FmContract))
2761 return false;
2762
2763 // Try combining even if fmul has more than one use as it eliminates
2764 // dependency between fadd(fsub) and fmul. However, it can extend liveranges
2765 // for fmul operands, so reject the transformation in register pressure
2766 // reduction mode.
2767 if (DoRegPressureReduce && !MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()))
2768 return false;
2769
2770 // Do not combine instructions from different basic blocks.
2771 if (Root.getParent() != MI->getParent())
2772 return false;
2773 return RISCV::hasEqualFRM(MI1: Root, MI2: *MI);
2774}
2775
2776static bool getFPFusedMultiplyPatterns(MachineInstr &Root,
2777 SmallVectorImpl<unsigned> &Patterns,
2778 bool DoRegPressureReduce) {
2779 unsigned Opc = Root.getOpcode();
2780 bool IsFAdd = isFADD(Opc);
2781 if (!IsFAdd && !isFSUB(Opc))
2782 return false;
2783 bool Added = false;
2784 if (canCombineFPFusedMultiply(Root, MO: Root.getOperand(i: 1),
2785 DoRegPressureReduce)) {
2786 Patterns.push_back(Elt: IsFAdd ? RISCVMachineCombinerPattern::FMADD_AX
2787 : RISCVMachineCombinerPattern::FMSUB);
2788 Added = true;
2789 }
2790 if (canCombineFPFusedMultiply(Root, MO: Root.getOperand(i: 2),
2791 DoRegPressureReduce)) {
2792 Patterns.push_back(Elt: IsFAdd ? RISCVMachineCombinerPattern::FMADD_XA
2793 : RISCVMachineCombinerPattern::FNMSUB);
2794 Added = true;
2795 }
2796 return Added;
2797}
2798
2799static bool getFPPatterns(MachineInstr &Root,
2800 SmallVectorImpl<unsigned> &Patterns,
2801 bool DoRegPressureReduce) {
2802 return getFPFusedMultiplyPatterns(Root, Patterns, DoRegPressureReduce);
2803}
2804
2805/// Utility routine that checks if \param MO is defined by an
2806/// \param CombineOpc instruction in the basic block \param MBB
2807static const MachineInstr *canCombine(const MachineBasicBlock &MBB,
2808 const MachineOperand &MO,
2809 unsigned CombineOpc) {
2810 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2811 const MachineInstr *MI = nullptr;
2812
2813 if (MO.isReg() && MO.getReg().isVirtual())
2814 MI = MRI.getUniqueVRegDef(Reg: MO.getReg());
2815 // And it needs to be in the trace (otherwise, it won't have a depth).
2816 if (!MI || MI->getParent() != &MBB || MI->getOpcode() != CombineOpc)
2817 return nullptr;
2818 // Must only used by the user we combine with.
2819 if (!MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()))
2820 return nullptr;
2821
2822 return MI;
2823}
2824
2825/// Utility routine that checks if \param MO is defined by a SLLI in \param
2826/// MBB that can be combined by splitting across 2 SHXADD instructions. The
2827/// first SHXADD shift amount is given by \param OuterShiftAmt.
2828static bool canCombineShiftIntoShXAdd(const MachineBasicBlock &MBB,
2829 const MachineOperand &MO,
2830 unsigned OuterShiftAmt) {
2831 const MachineInstr *ShiftMI = canCombine(MBB, MO, CombineOpc: RISCV::SLLI);
2832 if (!ShiftMI)
2833 return false;
2834
2835 unsigned InnerShiftAmt = ShiftMI->getOperand(i: 2).getImm();
2836 if (InnerShiftAmt < OuterShiftAmt || (InnerShiftAmt - OuterShiftAmt) > 3)
2837 return false;
2838
2839 return true;
2840}
2841
2842// Returns the shift amount from a SHXADD instruction. Returns 0 if the
2843// instruction is not a SHXADD.
2844static unsigned getSHXADDShiftAmount(unsigned Opc) {
2845 switch (Opc) {
2846 default:
2847 return 0;
2848 case RISCV::SH1ADD:
2849 return 1;
2850 case RISCV::SH2ADD:
2851 return 2;
2852 case RISCV::SH3ADD:
2853 return 3;
2854 }
2855}
2856
2857// Returns the shift amount from a SHXADD.UW instruction. Returns 0 if the
2858// instruction is not a SHXADD.UW.
2859static unsigned getSHXADDUWShiftAmount(unsigned Opc) {
2860 switch (Opc) {
2861 default:
2862 return 0;
2863 case RISCV::SH1ADD_UW:
2864 return 1;
2865 case RISCV::SH2ADD_UW:
2866 return 2;
2867 case RISCV::SH3ADD_UW:
2868 return 3;
2869 }
2870}
2871
2872// Look for opportunities to combine (sh3add Z, (add X, (slli Y, 5))) into
2873// (sh3add (sh2add Y, Z), X).
2874static bool getSHXADDPatterns(const MachineInstr &Root,
2875 SmallVectorImpl<unsigned> &Patterns) {
2876 unsigned ShiftAmt = getSHXADDShiftAmount(Opc: Root.getOpcode());
2877 if (!ShiftAmt)
2878 return false;
2879
2880 const MachineBasicBlock &MBB = *Root.getParent();
2881
2882 const MachineInstr *AddMI = canCombine(MBB, MO: Root.getOperand(i: 2), CombineOpc: RISCV::ADD);
2883 if (!AddMI)
2884 return false;
2885
2886 bool Found = false;
2887 if (canCombineShiftIntoShXAdd(MBB, MO: AddMI->getOperand(i: 1), OuterShiftAmt: ShiftAmt)) {
2888 Patterns.push_back(Elt: RISCVMachineCombinerPattern::SHXADD_ADD_SLLI_OP1);
2889 Found = true;
2890 }
2891 if (canCombineShiftIntoShXAdd(MBB, MO: AddMI->getOperand(i: 2), OuterShiftAmt: ShiftAmt)) {
2892 Patterns.push_back(Elt: RISCVMachineCombinerPattern::SHXADD_ADD_SLLI_OP2);
2893 Found = true;
2894 }
2895
2896 return Found;
2897}
2898
2899CombinerObjective RISCVInstrInfo::getCombinerObjective(unsigned Pattern) const {
2900 switch (Pattern) {
2901 case RISCVMachineCombinerPattern::FMADD_AX:
2902 case RISCVMachineCombinerPattern::FMADD_XA:
2903 case RISCVMachineCombinerPattern::FMSUB:
2904 case RISCVMachineCombinerPattern::FNMSUB:
2905 return CombinerObjective::MustReduceDepth;
2906 default:
2907 return TargetInstrInfo::getCombinerObjective(Pattern);
2908 }
2909}
2910
2911bool RISCVInstrInfo::getMachineCombinerPatterns(
2912 MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns,
2913 bool DoRegPressureReduce) const {
2914
2915 if (getFPPatterns(Root, Patterns, DoRegPressureReduce))
2916 return true;
2917
2918 if (getSHXADDPatterns(Root, Patterns))
2919 return true;
2920
2921 return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
2922 DoRegPressureReduce);
2923}
2924
2925static unsigned getFPFusedMultiplyOpcode(unsigned RootOpc, unsigned Pattern) {
2926 switch (RootOpc) {
2927 default:
2928 llvm_unreachable("Unexpected opcode");
2929 case RISCV::FADD_H:
2930 return RISCV::FMADD_H;
2931 case RISCV::FADD_S:
2932 return RISCV::FMADD_S;
2933 case RISCV::FADD_D:
2934 return RISCV::FMADD_D;
2935 case RISCV::FSUB_H:
2936 return Pattern == RISCVMachineCombinerPattern::FMSUB ? RISCV::FMSUB_H
2937 : RISCV::FNMSUB_H;
2938 case RISCV::FSUB_S:
2939 return Pattern == RISCVMachineCombinerPattern::FMSUB ? RISCV::FMSUB_S
2940 : RISCV::FNMSUB_S;
2941 case RISCV::FSUB_D:
2942 return Pattern == RISCVMachineCombinerPattern::FMSUB ? RISCV::FMSUB_D
2943 : RISCV::FNMSUB_D;
2944 }
2945}
2946
2947static unsigned getAddendOperandIdx(unsigned Pattern) {
2948 switch (Pattern) {
2949 default:
2950 llvm_unreachable("Unexpected pattern");
2951 case RISCVMachineCombinerPattern::FMADD_AX:
2952 case RISCVMachineCombinerPattern::FMSUB:
2953 return 2;
2954 case RISCVMachineCombinerPattern::FMADD_XA:
2955 case RISCVMachineCombinerPattern::FNMSUB:
2956 return 1;
2957 }
2958}
2959
2960static void combineFPFusedMultiply(MachineInstr &Root, MachineInstr &Prev,
2961 unsigned Pattern,
2962 SmallVectorImpl<MachineInstr *> &InsInstrs,
2963 SmallVectorImpl<MachineInstr *> &DelInstrs) {
2964 MachineFunction *MF = Root.getMF();
2965 MachineRegisterInfo &MRI = MF->getRegInfo();
2966 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
2967
2968 MachineOperand &Mul1 = Prev.getOperand(i: 1);
2969 MachineOperand &Mul2 = Prev.getOperand(i: 2);
2970 MachineOperand &Dst = Root.getOperand(i: 0);
2971 MachineOperand &Addend = Root.getOperand(i: getAddendOperandIdx(Pattern));
2972
2973 Register DstReg = Dst.getReg();
2974 unsigned FusedOpc = getFPFusedMultiplyOpcode(RootOpc: Root.getOpcode(), Pattern);
2975 uint32_t IntersectedFlags = Root.getFlags() & Prev.getFlags();
2976 DebugLoc MergedLoc =
2977 DILocation::getMergedLocation(LocA: Root.getDebugLoc(), LocB: Prev.getDebugLoc());
2978
2979 MachineInstrBuilder MIB = BuildMI(MF&: *MF, MIMD: MergedLoc, MCID: TII->get(Opcode: FusedOpc), DestReg: DstReg)
2980 .addReg(RegNo: Mul1.getReg())
2981 .addReg(RegNo: Mul2.getReg())
2982 .addReg(RegNo: Addend.getReg())
2983 .setMIFlags(IntersectedFlags);
2984
2985 InsInstrs.push_back(Elt: MIB);
2986 if (MRI.hasOneNonDBGUse(RegNo: Prev.getOperand(i: 0).getReg()))
2987 DelInstrs.push_back(Elt: &Prev);
2988 DelInstrs.push_back(Elt: &Root);
2989}
2990
2991// Combine patterns like (sh3add Z, (add X, (slli Y, 5))) to
2992// (sh3add (sh2add Y, Z), X) if the shift amount can be split across two
2993// shXadd instructions. The outer shXadd keeps its original opcode.
2994static void
2995genShXAddAddShift(MachineInstr &Root, unsigned AddOpIdx,
2996 SmallVectorImpl<MachineInstr *> &InsInstrs,
2997 SmallVectorImpl<MachineInstr *> &DelInstrs,
2998 DenseMap<Register, unsigned> &InstrIdxForVirtReg) {
2999 MachineFunction *MF = Root.getMF();
3000 MachineRegisterInfo &MRI = MF->getRegInfo();
3001 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
3002
3003 unsigned OuterShiftAmt = getSHXADDShiftAmount(Opc: Root.getOpcode());
3004 assert(OuterShiftAmt != 0 && "Unexpected opcode");
3005
3006 MachineInstr *AddMI = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: 2).getReg());
3007 MachineInstr *ShiftMI =
3008 MRI.getUniqueVRegDef(Reg: AddMI->getOperand(i: AddOpIdx).getReg());
3009
3010 unsigned InnerShiftAmt = ShiftMI->getOperand(i: 2).getImm();
3011 assert(InnerShiftAmt >= OuterShiftAmt && "Unexpected shift amount");
3012
3013 unsigned InnerOpc;
3014 switch (InnerShiftAmt - OuterShiftAmt) {
3015 default:
3016 llvm_unreachable("Unexpected shift amount");
3017 case 0:
3018 InnerOpc = RISCV::ADD;
3019 break;
3020 case 1:
3021 InnerOpc = RISCV::SH1ADD;
3022 break;
3023 case 2:
3024 InnerOpc = RISCV::SH2ADD;
3025 break;
3026 case 3:
3027 InnerOpc = RISCV::SH3ADD;
3028 break;
3029 }
3030
3031 const MachineOperand &X = AddMI->getOperand(i: 3 - AddOpIdx);
3032 const MachineOperand &Y = ShiftMI->getOperand(i: 1);
3033 const MachineOperand &Z = Root.getOperand(i: 1);
3034
3035 Register NewVR = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
3036
3037 auto MIB1 = BuildMI(MF&: *MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: InnerOpc), DestReg: NewVR)
3038 .addReg(RegNo: Y.getReg())
3039 .addReg(RegNo: Z.getReg());
3040 auto MIB2 = BuildMI(MF&: *MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: Root.getOpcode()),
3041 DestReg: Root.getOperand(i: 0).getReg())
3042 .addReg(RegNo: NewVR)
3043 .addReg(RegNo: X.getReg());
3044
3045 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
3046 InsInstrs.push_back(Elt: MIB1);
3047 InsInstrs.push_back(Elt: MIB2);
3048 DelInstrs.push_back(Elt: ShiftMI);
3049 DelInstrs.push_back(Elt: AddMI);
3050 DelInstrs.push_back(Elt: &Root);
3051}
3052
3053void RISCVInstrInfo::genAlternativeCodeSequence(
3054 MachineInstr &Root, unsigned Pattern,
3055 SmallVectorImpl<MachineInstr *> &InsInstrs,
3056 SmallVectorImpl<MachineInstr *> &DelInstrs,
3057 DenseMap<Register, unsigned> &InstrIdxForVirtReg) const {
3058 MachineRegisterInfo &MRI = Root.getMF()->getRegInfo();
3059 switch (Pattern) {
3060 default:
3061 TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs,
3062 DelInstrs, InstIdxForVirtReg&: InstrIdxForVirtReg);
3063 return;
3064 case RISCVMachineCombinerPattern::FMADD_AX:
3065 case RISCVMachineCombinerPattern::FMSUB: {
3066 MachineInstr &Prev = *MRI.getVRegDef(Reg: Root.getOperand(i: 1).getReg());
3067 combineFPFusedMultiply(Root, Prev, Pattern, InsInstrs, DelInstrs);
3068 return;
3069 }
3070 case RISCVMachineCombinerPattern::FMADD_XA:
3071 case RISCVMachineCombinerPattern::FNMSUB: {
3072 MachineInstr &Prev = *MRI.getVRegDef(Reg: Root.getOperand(i: 2).getReg());
3073 combineFPFusedMultiply(Root, Prev, Pattern, InsInstrs, DelInstrs);
3074 return;
3075 }
3076 case RISCVMachineCombinerPattern::SHXADD_ADD_SLLI_OP1:
3077 genShXAddAddShift(Root, AddOpIdx: 1, InsInstrs, DelInstrs, InstrIdxForVirtReg);
3078 return;
3079 case RISCVMachineCombinerPattern::SHXADD_ADD_SLLI_OP2:
3080 genShXAddAddShift(Root, AddOpIdx: 2, InsInstrs, DelInstrs, InstrIdxForVirtReg);
3081 return;
3082 }
3083}
3084
3085bool RISCVInstrInfo::verifyInstruction(const MachineInstr &MI,
3086 StringRef &ErrInfo) const {
3087 MCInstrDesc const &Desc = MI.getDesc();
3088
3089 for (const auto &[Index, Operand] : enumerate(First: Desc.operands())) {
3090 const MachineOperand &MO = MI.getOperand(i: Index);
3091 unsigned OpType = Operand.OperandType;
3092 switch (OpType) {
3093 default:
3094 if (OpType >= RISCVOp::OPERAND_FIRST_RISCV_IMM &&
3095 OpType <= RISCVOp::OPERAND_LAST_RISCV_IMM) {
3096 if (!MO.isImm()) {
3097 ErrInfo = "Expected an immediate operand.";
3098 return false;
3099 }
3100 int64_t Imm = MO.getImm();
3101 bool Ok;
3102 switch (OpType) {
3103 default:
3104 llvm_unreachable("Unexpected operand type");
3105
3106#define CASE_OPERAND_UIMM(NUM) \
3107 case RISCVOp::OPERAND_UIMM##NUM: \
3108 Ok = isUInt<NUM>(Imm); \
3109 break;
3110#define CASE_OPERAND_UIMM_LSB_ZEROS(BITS, SUFFIX) \
3111 case RISCVOp::OPERAND_UIMM##BITS##_LSB##SUFFIX: { \
3112 constexpr size_t NumZeros = sizeof(#SUFFIX) - 1; \
3113 Ok = isShiftedUInt<BITS - NumZeros, NumZeros>(Imm); \
3114 break; \
3115 }
3116#define CASE_OPERAND_SIMM(NUM) \
3117 case RISCVOp::OPERAND_SIMM##NUM: \
3118 Ok = isInt<NUM>(Imm); \
3119 break;
3120 // clang-format off
3121 CASE_OPERAND_UIMM(1)
3122 CASE_OPERAND_UIMM(2)
3123 CASE_OPERAND_UIMM(3)
3124 CASE_OPERAND_UIMM(4)
3125 CASE_OPERAND_UIMM(5)
3126 CASE_OPERAND_UIMM(6)
3127 CASE_OPERAND_UIMM(7)
3128 CASE_OPERAND_UIMM(8)
3129 CASE_OPERAND_UIMM(9)
3130 CASE_OPERAND_UIMM(10)
3131 CASE_OPERAND_UIMM(12)
3132 CASE_OPERAND_UIMM(16)
3133 CASE_OPERAND_UIMM(32)
3134 CASE_OPERAND_UIMM(48)
3135 CASE_OPERAND_UIMM(64)
3136 CASE_OPERAND_UIMM_LSB_ZEROS(2, 0)
3137 CASE_OPERAND_UIMM_LSB_ZEROS(5, 0)
3138 CASE_OPERAND_UIMM_LSB_ZEROS(6, 0)
3139 CASE_OPERAND_UIMM_LSB_ZEROS(6, 000)
3140 CASE_OPERAND_UIMM_LSB_ZEROS(7, 00)
3141 CASE_OPERAND_UIMM_LSB_ZEROS(7, 000)
3142 CASE_OPERAND_UIMM_LSB_ZEROS(8, 00)
3143 CASE_OPERAND_UIMM_LSB_ZEROS(8, 000)
3144 CASE_OPERAND_UIMM_LSB_ZEROS(9, 000)
3145 // clang-format on
3146 case RISCVOp::OPERAND_UIMM4_PLUS1:
3147 Ok = Imm >= 1 && Imm <= 16;
3148 break;
3149 case RISCVOp::OPERAND_UIMM5_NONZERO:
3150 Ok = isUInt<5>(x: Imm) && (Imm != 0);
3151 break;
3152 case RISCVOp::OPERAND_UIMM5_GT3:
3153 Ok = isUInt<5>(x: Imm) && (Imm > 3);
3154 break;
3155 case RISCVOp::OPERAND_UIMM5_PLUS1:
3156 Ok = Imm >= 1 && Imm <= 32;
3157 break;
3158 case RISCVOp::OPERAND_UIMM6_PLUS1:
3159 Ok = Imm >= 1 && Imm <= 64;
3160 break;
3161 case RISCVOp::OPERAND_UIMM8_GE32:
3162 Ok = isUInt<8>(x: Imm) && Imm >= 32;
3163 break;
3164 case RISCVOp::OPERAND_UIMM9_YBNDSWI:
3165 Ok = RISCV::isValidYBNDSWImm(Imm);
3166 break;
3167 case RISCVOp::OPERAND_SIMM10_LSB0000_NONZERO:
3168 Ok = isShiftedInt<6, 4>(x: Imm) && (Imm != 0);
3169 break;
3170 case RISCVOp::OPERAND_UIMM10_LSB00_NONZERO:
3171 Ok = isShiftedUInt<8, 2>(x: Imm) && (Imm != 0);
3172 break;
3173 case RISCVOp::OPERAND_UIMM16_NONZERO:
3174 Ok = isUInt<16>(x: Imm) && (Imm != 0);
3175 break;
3176 case RISCVOp::OPERAND_THREE:
3177 Ok = Imm == 3;
3178 break;
3179 case RISCVOp::OPERAND_FOUR:
3180 Ok = Imm == 4;
3181 break;
3182 case RISCVOp::OPERAND_IMM5_ZIBI:
3183 Ok = (isUInt<5>(x: Imm) && Imm != 0) || Imm == -1;
3184 break;
3185 // clang-format off
3186 CASE_OPERAND_SIMM(5)
3187 CASE_OPERAND_SIMM(6)
3188 CASE_OPERAND_SIMM(8)
3189 CASE_OPERAND_SIMM(10)
3190 CASE_OPERAND_SIMM(11)
3191 CASE_OPERAND_SIMM(12)
3192 CASE_OPERAND_SIMM(26)
3193 // clang-format on
3194 case RISCVOp::OPERAND_SIMM5_PLUS1:
3195 Ok = Imm >= -15 && Imm <= 16;
3196 break;
3197 case RISCVOp::OPERAND_SIMM5_NONZERO:
3198 Ok = isInt<5>(x: Imm) && (Imm != 0);
3199 break;
3200 case RISCVOp::OPERAND_SIMM6_NONZERO:
3201 Ok = Imm != 0 && isInt<6>(x: Imm);
3202 break;
3203 case RISCVOp::OPERAND_VTYPEI10:
3204 Ok = isUInt<10>(x: Imm) && RISCVVType::isValidVType(VType: Imm);
3205 break;
3206 case RISCVOp::OPERAND_VTYPEI11:
3207 Ok = isUInt<11>(x: Imm) && RISCVVType::isValidVType(VType: Imm);
3208 break;
3209 case RISCVOp::OPERAND_SIMM12_LSB00000:
3210 Ok = isShiftedInt<7, 5>(x: Imm);
3211 break;
3212 case RISCVOp::OPERAND_SIMM16_NONZERO:
3213 Ok = isInt<16>(x: Imm) && (Imm != 0);
3214 break;
3215 case RISCVOp::OPERAND_SIMM20_LI:
3216 Ok = isInt<20>(x: Imm);
3217 break;
3218 case RISCVOp::OPERAND_UIMMLOG2XLEN:
3219 Ok = STI.is64Bit() ? isUInt<6>(x: Imm) : isUInt<5>(x: Imm);
3220 break;
3221 case RISCVOp::OPERAND_UIMMLOG2XLEN_NONZERO:
3222 Ok = STI.is64Bit() ? isUInt<6>(x: Imm) : isUInt<5>(x: Imm);
3223 Ok = Ok && Imm != 0;
3224 break;
3225 case RISCVOp::OPERAND_CLUI_IMM:
3226 Ok = (isUInt<5>(x: Imm) && Imm != 0) || (Imm >= 0xfffe0 && Imm <= 0xfffff);
3227 break;
3228 case RISCVOp::OPERAND_RVKRNUM:
3229 Ok = Imm >= 0 && Imm <= 10;
3230 break;
3231 case RISCVOp::OPERAND_RVKRNUM_0_7:
3232 Ok = Imm >= 0 && Imm <= 7;
3233 break;
3234 case RISCVOp::OPERAND_RVKRNUM_1_10:
3235 Ok = Imm >= 1 && Imm <= 10;
3236 break;
3237 case RISCVOp::OPERAND_RVKRNUM_2_14:
3238 Ok = Imm >= 2 && Imm <= 14;
3239 break;
3240 case RISCVOp::OPERAND_RLIST:
3241 Ok = Imm >= RISCVZC::RA && Imm <= RISCVZC::RA_S0_S11;
3242 break;
3243 case RISCVOp::OPERAND_RLIST_S0:
3244 Ok = Imm >= RISCVZC::RA_S0 && Imm <= RISCVZC::RA_S0_S11;
3245 break;
3246 case RISCVOp::OPERAND_STACKADJ:
3247 Ok = Imm >= 0 && Imm <= 48 && Imm % 16 == 0;
3248 break;
3249 case RISCVOp::OPERAND_FRMARG:
3250 Ok = RISCVFPRndMode::isValidRoundingMode(Mode: Imm);
3251 break;
3252 case RISCVOp::OPERAND_RTZARG:
3253 Ok = Imm == RISCVFPRndMode::RTZ;
3254 break;
3255 case RISCVOp::OPERAND_SMTVType:
3256 Ok = XSMTVTypeMode::isValidSMTVTypeMode(Mode: Imm);
3257 break;
3258 case RISCVOp::OPERAND_SMTI8:
3259 Ok = Imm == XSMTVTypeMode::SMT_I8;
3260 break;
3261 case RISCVOp::OPERAND_COND_CODE:
3262 Ok = Imm >= 0 && Imm < RISCVCC::COND_INVALID;
3263 break;
3264 case RISCVOp::OPERAND_ATOMIC_ORDERING:
3265 Ok = isValidAtomicOrdering(I: Imm);
3266 break;
3267 case RISCVOp::OPERAND_VEC_POLICY:
3268 Ok = (Imm & (RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC)) ==
3269 Imm;
3270 break;
3271 case RISCVOp::OPERAND_SEW:
3272 Ok = (isUInt<5>(x: Imm) && RISCVVType::isValidSEW(SEW: 1 << Imm));
3273 break;
3274 case RISCVOp::OPERAND_SEW_MASK:
3275 Ok = Imm == 0;
3276 break;
3277 case RISCVOp::OPERAND_VEC_RM:
3278 assert(RISCVII::hasRoundModeOp(Desc.TSFlags));
3279 if (RISCVII::usesVXRM(TSFlags: Desc.TSFlags))
3280 Ok = isUInt<2>(x: Imm);
3281 else
3282 Ok = RISCVFPRndMode::isValidRoundingMode(Mode: Imm);
3283 break;
3284 case RISCVOp::OPERAND_XSFMM_VTYPE:
3285 Ok = RISCVVType::isValidXSfmmVType(VTypeI: Imm);
3286 break;
3287 case RISCVOp::OPERAND_XSFMM_TWIDEN:
3288 Ok = Imm == 1 || Imm == 2 || Imm == 4;
3289 break;
3290 }
3291 if (!Ok) {
3292 ErrInfo = "Invalid immediate";
3293 return false;
3294 }
3295 }
3296 break;
3297 case RISCVOp::OPERAND_SIMM12_LO:
3298 // TODO: We could be stricter about what non-register operands are
3299 // allowed.
3300 if (MO.isReg()) {
3301 ErrInfo = "Expected a non-register operand.";
3302 return false;
3303 }
3304 if (MO.isImm() && !isInt<12>(x: MO.getImm())) {
3305 ErrInfo = "Invalid immediate";
3306 return false;
3307 }
3308 break;
3309 case RISCVOp::OPERAND_UIMM20_LUI:
3310 case RISCVOp::OPERAND_UIMM20_AUIPC:
3311 // TODO: We could be stricter about what non-register operands are
3312 // allowed.
3313 if (MO.isReg()) {
3314 ErrInfo = "Expected a non-register operand.";
3315 return false;
3316 }
3317 if (MO.isImm() && !isUInt<20>(x: MO.getImm())) {
3318 ErrInfo = "Invalid immediate";
3319 return false;
3320 }
3321 break;
3322 case RISCVOp::OPERAND_BARE_SIMM32:
3323 // TODO: We could be stricter about what non-register operands are
3324 // allowed.
3325 if (MO.isReg()) {
3326 ErrInfo = "Expected a non-register operand.";
3327 return false;
3328 }
3329 if (MO.isImm() && !isInt<32>(x: MO.getImm())) {
3330 ErrInfo = "Invalid immediate";
3331 return false;
3332 }
3333 break;
3334 case RISCVOp::OPERAND_AVL:
3335 if (MO.isImm()) {
3336 int64_t Imm = MO.getImm();
3337 // VLMAX is represented as -1.
3338 if (!isUInt<5>(x: Imm) && Imm != -1) {
3339 ErrInfo = "Invalid immediate";
3340 return false;
3341 }
3342 } else if (!MO.isReg()) {
3343 ErrInfo = "Expected a register or immediate operand.";
3344 return false;
3345 }
3346 break;
3347 case RISCVOp::OPERAND_SFB_RHS:
3348 if (!MO.isReg() && !MO.isImm()) {
3349 ErrInfo = "Expected a register or immediate operand.";
3350 return false;
3351 }
3352 break;
3353 }
3354 }
3355
3356 const uint64_t TSFlags = Desc.TSFlags;
3357 if (RISCVII::hasVLOp(TSFlags)) {
3358 const MachineOperand &Op = MI.getOperand(i: RISCVII::getVLOpNum(Desc));
3359 if (!Op.isImm() && !Op.isReg()) {
3360 ErrInfo = "Invalid operand type for VL operand";
3361 return false;
3362 }
3363 if (Op.isReg() && Op.getReg().isValid()) {
3364 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
3365 auto *RC = MRI.getRegClass(Reg: Op.getReg());
3366 if (!RISCV::GPRNoX0RegClass.hasSubClassEq(RC)) {
3367 ErrInfo = "Invalid register class for VL operand";
3368 return false;
3369 }
3370 }
3371 if (!RISCVII::hasSEWOp(TSFlags)) {
3372 ErrInfo = "VL operand w/o SEW operand?";
3373 return false;
3374 }
3375 }
3376 if (RISCVII::hasSEWOp(TSFlags)) {
3377 unsigned OpIdx = RISCVII::getSEWOpNum(Desc);
3378 if (!MI.getOperand(i: OpIdx).isImm()) {
3379 ErrInfo = "SEW value expected to be an immediate";
3380 return false;
3381 }
3382 uint64_t Log2SEW = MI.getOperand(i: OpIdx).getImm();
3383 if (Log2SEW > 31) {
3384 ErrInfo = "Unexpected SEW value";
3385 return false;
3386 }
3387 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
3388 if (!RISCVVType::isValidSEW(SEW)) {
3389 ErrInfo = "Unexpected SEW value";
3390 return false;
3391 }
3392 }
3393 if (RISCVII::hasVecPolicyOp(TSFlags)) {
3394 unsigned OpIdx = RISCVII::getVecPolicyOpNum(Desc);
3395 if (!MI.getOperand(i: OpIdx).isImm()) {
3396 ErrInfo = "Policy operand expected to be an immediate";
3397 return false;
3398 }
3399 uint64_t Policy = MI.getOperand(i: OpIdx).getImm();
3400 if (Policy > (RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC)) {
3401 ErrInfo = "Invalid Policy Value";
3402 return false;
3403 }
3404 if (!RISCVII::hasVLOp(TSFlags)) {
3405 ErrInfo = "policy operand w/o VL operand?";
3406 return false;
3407 }
3408
3409 // VecPolicy operands can only exist on instructions with passthru/merge
3410 // arguments. Note that not all arguments with passthru have vec policy
3411 // operands- some instructions have implicit policies.
3412 unsigned UseOpIdx;
3413 if (!MI.isRegTiedToUseOperand(DefOpIdx: 0, UseOpIdx: &UseOpIdx)) {
3414 ErrInfo = "policy operand w/o tied operand?";
3415 return false;
3416 }
3417 }
3418
3419 if (int Idx = RISCVII::getFRMOpNum(Desc);
3420 Idx >= 0 && MI.getOperand(i: Idx).getImm() == RISCVFPRndMode::DYN &&
3421 !MI.readsRegister(Reg: RISCV::FRM, /*TRI=*/nullptr)) {
3422 ErrInfo = "dynamic rounding mode should read FRM";
3423 return false;
3424 }
3425
3426 return true;
3427}
3428
3429bool RISCVInstrInfo::canFoldIntoAddrMode(const MachineInstr &MemI, Register Reg,
3430 const MachineInstr &AddrI,
3431 ExtAddrMode &AM) const {
3432 switch (MemI.getOpcode()) {
3433 default:
3434 return false;
3435 case RISCV::LB:
3436 case RISCV::LBU:
3437 case RISCV::LH:
3438 case RISCV::LH_INX:
3439 case RISCV::LHU:
3440 case RISCV::LW:
3441 case RISCV::LW_INX:
3442 case RISCV::LWU:
3443 case RISCV::LD:
3444 case RISCV::LD_RV32:
3445 case RISCV::FLH:
3446 case RISCV::FLW:
3447 case RISCV::FLD:
3448 case RISCV::SB:
3449 case RISCV::SH:
3450 case RISCV::SH_INX:
3451 case RISCV::SW:
3452 case RISCV::SW_INX:
3453 case RISCV::SD:
3454 case RISCV::SD_RV32:
3455 case RISCV::FSH:
3456 case RISCV::FSW:
3457 case RISCV::FSD:
3458 break;
3459 }
3460
3461 if (MemI.getOperand(i: 0).getReg() == Reg)
3462 return false;
3463
3464 if (AddrI.getOpcode() != RISCV::ADDI || !AddrI.getOperand(i: 1).isReg() ||
3465 !AddrI.getOperand(i: 2).isImm())
3466 return false;
3467
3468 int64_t OldOffset = MemI.getOperand(i: 2).getImm();
3469 int64_t Disp = AddrI.getOperand(i: 2).getImm();
3470 int64_t NewOffset = OldOffset + Disp;
3471 if (!STI.is64Bit())
3472 NewOffset = SignExtend64<32>(x: NewOffset);
3473
3474 if (!isInt<12>(x: NewOffset))
3475 return false;
3476
3477 AM.BaseReg = AddrI.getOperand(i: 1).getReg();
3478 AM.ScaledReg = 0;
3479 AM.Scale = 0;
3480 AM.Displacement = NewOffset;
3481 AM.Form = ExtAddrMode::Formula::Basic;
3482 return true;
3483}
3484
3485MachineInstr *RISCVInstrInfo::emitLdStWithAddr(MachineInstr &MemI,
3486 const ExtAddrMode &AM) const {
3487
3488 const DebugLoc &DL = MemI.getDebugLoc();
3489 MachineBasicBlock &MBB = *MemI.getParent();
3490
3491 assert(AM.ScaledReg == 0 && AM.Scale == 0 &&
3492 "Addressing mode not supported for folding");
3493
3494 return BuildMI(BB&: MBB, I&: MemI, MIMD: DL, MCID: get(Opcode: MemI.getOpcode()))
3495 .addReg(RegNo: MemI.getOperand(i: 0).getReg(), Flags: getDefRegState(B: MemI.mayLoad()))
3496 .addReg(RegNo: AM.BaseReg)
3497 .addImm(Val: AM.Displacement)
3498 .setMemRefs(MemI.memoperands())
3499 .setMIFlags(MemI.getFlags());
3500}
3501
3502// TODO: At the moment, MIPS introduced paring of instructions operating with
3503// word or double word. This should be extended with more instructions when more
3504// vendors support load/store pairing.
3505bool RISCVInstrInfo::isPairableLdStInstOpc(unsigned Opc) {
3506 switch (Opc) {
3507 default:
3508 return false;
3509 case RISCV::SW:
3510 case RISCV::SD:
3511 case RISCV::LD:
3512 case RISCV::LW:
3513 return true;
3514 }
3515}
3516
3517bool RISCVInstrInfo::isLdStSafeToPair(const MachineInstr &LdSt,
3518 const TargetRegisterInfo *TRI) {
3519 // If this is a volatile load/store, don't mess with it.
3520 if (LdSt.hasOrderedMemoryRef() || LdSt.getNumExplicitOperands() != 3)
3521 return false;
3522
3523 if (LdSt.getOperand(i: 1).isFI())
3524 return true;
3525
3526 assert(LdSt.getOperand(1).isReg() && "Expected a reg operand.");
3527 // Can't cluster if the instruction modifies the base register
3528 // or it is update form. e.g. ld x5,8(x5)
3529 if (LdSt.modifiesRegister(Reg: LdSt.getOperand(i: 1).getReg(), TRI))
3530 return false;
3531
3532 if (!LdSt.getOperand(i: 2).isImm())
3533 return false;
3534
3535 return true;
3536}
3537
3538bool RISCVInstrInfo::getMemOperandsWithOffsetWidth(
3539 const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
3540 int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width) const {
3541 if (!LdSt.mayLoadOrStore())
3542 return false;
3543
3544 // Conservatively, only handle scalar loads/stores for now.
3545 switch (LdSt.getOpcode()) {
3546 case RISCV::LB:
3547 case RISCV::LBU:
3548 case RISCV::SB:
3549 case RISCV::LH:
3550 case RISCV::LH_INX:
3551 case RISCV::LHU:
3552 case RISCV::FLH:
3553 case RISCV::SH:
3554 case RISCV::SH_INX:
3555 case RISCV::FSH:
3556 case RISCV::LW:
3557 case RISCV::LW_INX:
3558 case RISCV::LWU:
3559 case RISCV::FLW:
3560 case RISCV::SW:
3561 case RISCV::SW_INX:
3562 case RISCV::FSW:
3563 case RISCV::LD:
3564 case RISCV::LD_RV32:
3565 case RISCV::FLD:
3566 case RISCV::SD:
3567 case RISCV::SD_RV32:
3568 case RISCV::FSD:
3569 break;
3570 default:
3571 return false;
3572 }
3573 const MachineOperand *BaseOp;
3574 OffsetIsScalable = false;
3575 if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width))
3576 return false;
3577 BaseOps.push_back(Elt: BaseOp);
3578 return true;
3579}
3580
3581// TODO: This was copied from SIInstrInfo. Could it be lifted to a common
3582// helper?
3583static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
3584 ArrayRef<const MachineOperand *> BaseOps1,
3585 const MachineInstr &MI2,
3586 ArrayRef<const MachineOperand *> BaseOps2) {
3587 // Only examine the first "base" operand of each instruction, on the
3588 // assumption that it represents the real base address of the memory access.
3589 // Other operands are typically offsets or indices from this base address.
3590 if (BaseOps1.front()->isIdenticalTo(Other: *BaseOps2.front()))
3591 return true;
3592
3593 if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
3594 return false;
3595
3596 auto MO1 = *MI1.memoperands_begin();
3597 auto MO2 = *MI2.memoperands_begin();
3598 if (MO1->getAddrSpace() != MO2->getAddrSpace())
3599 return false;
3600
3601 auto Base1 = MO1->getValue();
3602 auto Base2 = MO2->getValue();
3603 if (!Base1 || !Base2)
3604 return false;
3605 Base1 = getUnderlyingObject(V: Base1);
3606 Base2 = getUnderlyingObject(V: Base2);
3607
3608 if (isa<UndefValue>(Val: Base1) || isa<UndefValue>(Val: Base2))
3609 return false;
3610
3611 return Base1 == Base2;
3612}
3613
3614bool RISCVInstrInfo::shouldClusterMemOps(
3615 ArrayRef<const MachineOperand *> BaseOps1, int64_t Offset1,
3616 bool OffsetIsScalable1, ArrayRef<const MachineOperand *> BaseOps2,
3617 int64_t Offset2, bool OffsetIsScalable2, unsigned ClusterSize,
3618 unsigned NumBytes) const {
3619 // If the mem ops (to be clustered) do not have the same base ptr, then they
3620 // should not be clustered
3621 if (!BaseOps1.empty() && !BaseOps2.empty()) {
3622 const MachineInstr &FirstLdSt = *BaseOps1.front()->getParent();
3623 const MachineInstr &SecondLdSt = *BaseOps2.front()->getParent();
3624 if (!memOpsHaveSameBasePtr(MI1: FirstLdSt, BaseOps1, MI2: SecondLdSt, BaseOps2))
3625 return false;
3626 } else if (!BaseOps1.empty() || !BaseOps2.empty()) {
3627 // If only one base op is empty, they do not have the same base ptr
3628 return false;
3629 }
3630
3631 unsigned CacheLineSize =
3632 BaseOps1.front()->getParent()->getMF()->getSubtarget().getCacheLineSize();
3633 // Assume a cache line size of 64 bytes if no size is set in RISCVSubtarget.
3634 CacheLineSize = CacheLineSize ? CacheLineSize : 64;
3635 // Cluster if the memory operations are on the same or a neighbouring cache
3636 // line, but limit the maximum ClusterSize to avoid creating too much
3637 // additional register pressure.
3638 return ClusterSize <= 4 && std::abs(i: Offset1 - Offset2) < CacheLineSize;
3639}
3640
3641// Set BaseReg (the base register operand), Offset (the byte offset being
3642// accessed) and the access Width of the passed instruction that reads/writes
3643// memory. Returns false if the instruction does not read/write memory or the
3644// BaseReg/Offset/Width can't be determined. Is not guaranteed to always
3645// recognise base operands and offsets in all cases.
3646// TODO: Add an IsScalable bool ref argument (like the equivalent AArch64
3647// function) and set it as appropriate.
3648bool RISCVInstrInfo::getMemOperandWithOffsetWidth(
3649 const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset,
3650 LocationSize &Width) const {
3651 if (!LdSt.mayLoadOrStore())
3652 return false;
3653
3654 // Here we assume the standard RISC-V ISA, which uses a base+offset
3655 // addressing mode. You'll need to relax these conditions to support custom
3656 // load/store instructions.
3657 if (LdSt.getNumExplicitOperands() != 3)
3658 return false;
3659 if ((!LdSt.getOperand(i: 1).isReg() && !LdSt.getOperand(i: 1).isFI()) ||
3660 !LdSt.getOperand(i: 2).isImm())
3661 return false;
3662
3663 if (!LdSt.hasOneMemOperand())
3664 return false;
3665
3666 Width = (*LdSt.memoperands_begin())->getSize();
3667 BaseReg = &LdSt.getOperand(i: 1);
3668 Offset = LdSt.getOperand(i: 2).getImm();
3669 return true;
3670}
3671
3672bool RISCVInstrInfo::areMemAccessesTriviallyDisjoint(
3673 const MachineInstr &MIa, const MachineInstr &MIb) const {
3674 assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
3675 assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
3676
3677 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() ||
3678 MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
3679 return false;
3680
3681 // Retrieve the base register, offset from the base register and width. Width
3682 // is the size of memory that is being loaded/stored (e.g. 1, 2, 4). If
3683 // base registers are identical, and the offset of a lower memory access +
3684 // the width doesn't overlap the offset of a higher memory access,
3685 // then the memory accesses are different.
3686 const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
3687 int64_t OffsetA = 0, OffsetB = 0;
3688 LocationSize WidthA = LocationSize::precise(Value: 0),
3689 WidthB = LocationSize::precise(Value: 0);
3690 if (getMemOperandWithOffsetWidth(LdSt: MIa, BaseReg&: BaseOpA, Offset&: OffsetA, Width&: WidthA) &&
3691 getMemOperandWithOffsetWidth(LdSt: MIb, BaseReg&: BaseOpB, Offset&: OffsetB, Width&: WidthB)) {
3692 if (BaseOpA->isIdenticalTo(Other: *BaseOpB)) {
3693 int LowOffset = std::min(a: OffsetA, b: OffsetB);
3694 int HighOffset = std::max(a: OffsetA, b: OffsetB);
3695 LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
3696 if (LowWidth.hasValue() &&
3697 LowOffset + (int)LowWidth.getValue() <= HighOffset)
3698 return true;
3699 }
3700 }
3701 return false;
3702}
3703
3704std::pair<unsigned, unsigned>
3705RISCVInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
3706 const unsigned Mask = RISCVII::MO_DIRECT_FLAG_MASK;
3707 return std::make_pair(x: TF & Mask, y: TF & ~Mask);
3708}
3709
3710ArrayRef<std::pair<unsigned, const char *>>
3711RISCVInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
3712 using namespace RISCVII;
3713 static const std::pair<unsigned, const char *> TargetFlags[] = {
3714 {MO_CALL, "riscv-call"},
3715 {MO_LO, "riscv-lo"},
3716 {MO_HI, "riscv-hi"},
3717 {MO_PCREL_LO, "riscv-pcrel-lo"},
3718 {MO_PCREL_HI, "riscv-pcrel-hi"},
3719 {MO_GOT_HI, "riscv-got-hi"},
3720 {MO_TPREL_LO, "riscv-tprel-lo"},
3721 {MO_TPREL_HI, "riscv-tprel-hi"},
3722 {MO_TPREL_ADD, "riscv-tprel-add"},
3723 {MO_TLS_GOT_HI, "riscv-tls-got-hi"},
3724 {MO_TLS_GD_HI, "riscv-tls-gd-hi"},
3725 {MO_TLSDESC_HI, "riscv-tlsdesc-hi"},
3726 {MO_TLSDESC_LOAD_LO, "riscv-tlsdesc-load-lo"},
3727 {MO_TLSDESC_ADD_LO, "riscv-tlsdesc-add-lo"},
3728 {MO_TLSDESC_CALL, "riscv-tlsdesc-call"},
3729 {MO_QC_ACCESS, "riscv-qc-access"},
3730 };
3731 return ArrayRef(TargetFlags);
3732}
3733
3734bool RISCVInstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB,
3735 unsigned &Flags) const {
3736 // More accurate safety checking is done in getOutliningCandidateInfo.
3737 return TargetInstrInfo::isMBBSafeToOutlineFrom(MBB, Flags);
3738}
3739
3740// Enum values indicating how an outlined call should be constructed.
3741enum MachineOutlinerConstructionID {
3742 MachineOutlinerTailCall,
3743 MachineOutlinerDefault,
3744 MachineOutlinerRegSave
3745};
3746
3747bool RISCVInstrInfo::shouldOutlineFromFunctionByDefault(
3748 MachineFunction &MF) const {
3749 return MF.getFunction().hasMinSize();
3750}
3751
3752static bool isCandidatePatchable(const MachineBasicBlock &MBB) {
3753 const MachineFunction *MF = MBB.getParent();
3754 const Function &F = MF->getFunction();
3755 return F.getFnAttribute(Kind: "fentry-call").getValueAsBool() ||
3756 F.hasFnAttribute(Kind: "patchable-function-entry");
3757}
3758
3759static bool isMIReadsReg(const MachineInstr &MI, const TargetRegisterInfo *TRI,
3760 MCRegister RegNo) {
3761 return MI.readsRegister(Reg: RegNo, TRI) ||
3762 MI.getDesc().hasImplicitUseOfPhysReg(Reg: RegNo);
3763}
3764
3765static bool isMIModifiesReg(const MachineInstr &MI,
3766 const TargetRegisterInfo *TRI, MCRegister RegNo) {
3767 return MI.modifiesRegister(Reg: RegNo, TRI) ||
3768 MI.getDesc().hasImplicitDefOfPhysReg(Reg: RegNo);
3769}
3770
3771static bool cannotInsertTailCall(const MachineBasicBlock &MBB) {
3772 if (!MBB.back().isReturn())
3773 return true;
3774 if (isCandidatePatchable(MBB))
3775 return true;
3776
3777 // If the candidate reads the pre-set register
3778 // that can be used for expanding PseudoTAIL instruction,
3779 // then we cannot insert tail call.
3780 const TargetSubtargetInfo &STI = MBB.getParent()->getSubtarget();
3781 const RISCVMachineFunctionInfo *RVFI =
3782 MBB.getParent()->getInfo<RISCVMachineFunctionInfo>();
3783 // When cf-protection-branch is active, the outliner will emit PseudoTAILX7
3784 // which always uses X7. Otherwise, PseudoTAIL is emitted and the register
3785 // is determined by Zicfilp at encode time.
3786 MCRegister TailExpandUseRegNo =
3787 RVFI->hasCFProtectionBranch()
3788 ? RISCV::X7
3789 : RISCVII::getTailExpandUseRegNo(FeatureBits: STI.getFeatureBits());
3790 for (const MachineInstr &MI : MBB) {
3791 if (isMIReadsReg(MI, TRI: STI.getRegisterInfo(), RegNo: TailExpandUseRegNo))
3792 return true;
3793 if (isMIModifiesReg(MI, TRI: STI.getRegisterInfo(), RegNo: TailExpandUseRegNo))
3794 break;
3795 }
3796 return false;
3797}
3798
3799static Register findRegisterToSaveX5To(outliner::Candidate &C,
3800 const TargetRegisterInfo &TRI) {
3801 // Candidate registers for saving X5: t1-t6
3802 static const MCPhysReg TempRegs[] = {
3803 RISCV::X6, // t1
3804 RISCV::X7, // t2
3805 RISCV::X28, // t3
3806 RISCV::X29, // t4
3807 RISCV::X30, // t5
3808 RISCV::X31 // t6
3809 };
3810
3811 const MachineFunction *MF = C.getMF();
3812 const MachineRegisterInfo &MRI = MF->getRegInfo();
3813
3814 for (MCPhysReg Reg : TempRegs) {
3815 if (MRI.isReserved(PhysReg: Reg))
3816 continue;
3817
3818 if (C.isAvailableAcrossAndOutOfSeq(Reg, TRI) &&
3819 C.isAvailableInsideSeq(Reg, TRI)) {
3820 return Reg;
3821 }
3822 }
3823
3824 return Register();
3825}
3826
3827bool RISCVInstrInfo::analyzeCandidate(outliner::Candidate &C) const {
3828 // If the expansion register for tail calls is live across the candidate
3829 // outlined call site, we cannot outline that candidate as the expansion
3830 // would clobber the register.
3831 const RISCVMachineFunctionInfo *RVFI =
3832 C.getMF()->getInfo<RISCVMachineFunctionInfo>();
3833 MCRegister TailExpandUseReg =
3834 RVFI->hasCFProtectionBranch()
3835 ? RISCV::X7
3836 : RISCVII::getTailExpandUseRegNo(FeatureBits: STI.getFeatureBits());
3837 if (C.back().isReturn() &&
3838 !C.isAvailableAcrossAndOutOfSeq(Reg: TailExpandUseReg, TRI: RegInfo)) {
3839 LLVM_DEBUG(dbgs() << "MBB:\n" << *C.getMBB());
3840 LLVM_DEBUG(dbgs() << "Cannot be outlined between: " << C.front() << "and "
3841 << C.back());
3842 LLVM_DEBUG(dbgs() << "Because the tail-call register is live across "
3843 "the proposed outlined function call\n");
3844 return true;
3845 }
3846
3847 // If last instruction is return then we can rely on
3848 // the verification already performed in the getOutliningTypeImpl.
3849 if (C.back().isReturn()) {
3850 assert(!cannotInsertTailCall(*C.getMBB()) &&
3851 "The candidate who uses return instruction must be outlined "
3852 "using tail call");
3853 return false;
3854 }
3855
3856 // Filter out candidates where the X5 register (t0) can't be used to setup
3857 // the function call.
3858 if (!C.isAvailableInsideSeq(Reg: RISCV::X5, TRI: RegInfo))
3859 return true;
3860
3861 // If X5 is available in the region, use X5 directly (MachineOutlinerDefault).
3862 if (C.isAvailableAcrossAndOutOfSeq(Reg: RISCV::X5, TRI: RegInfo))
3863 return false;
3864
3865 // Otherwise, try to save X5 into t1-t6 (MachineOutlinerRegSave).
3866 if (STI.getCLOpts().outliner_regsave && !C.getMF()->isOutlined() &&
3867 findRegisterToSaveX5To(C, TRI: RegInfo))
3868 return false;
3869
3870 return true;
3871}
3872
3873std::optional<std::unique_ptr<outliner::OutlinedFunction>>
3874RISCVInstrInfo::getOutliningCandidateInfo(
3875 const MachineModuleInfo &MMI,
3876 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
3877 unsigned MinRepeats) const {
3878
3879 // Analyze each candidate and erase the ones that are not viable.
3880 llvm::erase_if(C&: RepeatedSequenceLocs, P: [this](auto Candidate) {
3881 return analyzeCandidate(C&: Candidate);
3882 });
3883
3884 // If the sequence doesn't have enough candidates left, then we're done.
3885 if (RepeatedSequenceLocs.size() < MinRepeats)
3886 return std::nullopt;
3887
3888 // Each RepeatedSequenceLoc is identical.
3889 outliner::Candidate &Candidate = RepeatedSequenceLocs[0];
3890 unsigned InstrSizeCExt =
3891 Candidate.getMF()->getSubtarget<RISCVSubtarget>().hasStdExtZca() ? 2 : 4;
3892 unsigned CallOverhead = 0, FrameOverhead = 0;
3893
3894 // Count the number of CFI instructions in the candidate, if present.
3895 unsigned CFICount = 0;
3896 for (auto &I : Candidate) {
3897 if (I.isCFIInstruction())
3898 CFICount++;
3899 }
3900
3901 // Ensure CFI coverage matches: comparing the number of CFIs in the candidate
3902 // with the total number of CFIs in the parent function for each candidate.
3903 // Outlining only a subset of a function’s CFIs would split the unwind state
3904 // across two code regions and lead to incorrect address offsets between the
3905 // outlined body and the remaining code. To preserve correct unwind info, we
3906 // only outline when all CFIs in the function can be outlined together.
3907 for (outliner::Candidate &C : RepeatedSequenceLocs) {
3908 std::vector<MCCFIInstruction> CFIInstructions =
3909 C.getMF()->getFrameInstructions();
3910
3911 if (CFICount > 0 && CFICount != CFIInstructions.size())
3912 return std::nullopt;
3913 }
3914
3915 MachineOutlinerConstructionID MOCI = MachineOutlinerDefault;
3916 if (Candidate.back().isReturn()) {
3917 MOCI = MachineOutlinerTailCall;
3918 // tail call = auipc + jalr in the worst case without linker relaxation.
3919 // FIXME: This code suggests the JALR can be compressed - how?
3920 CallOverhead = 4 + InstrSizeCExt;
3921 // Using tail call we move ret instruction from caller to callee.
3922 FrameOverhead = 0;
3923 } else {
3924 // call t0, function = 8 bytes.
3925 CallOverhead = 8;
3926 // jr t0 = 4 bytes, 2 bytes if compressed instructions are enabled.
3927 FrameOverhead = InstrSizeCExt;
3928 }
3929
3930 // If we have CFI instructions, we can only outline if the outlined section
3931 // can be a tail call.
3932 if (MOCI != MachineOutlinerTailCall && CFICount > 0)
3933 return std::nullopt;
3934
3935 if (STI.getCLOpts().outliner_regsave && MOCI == MachineOutlinerDefault) {
3936 // Set per-candidate overhead based on X5 availability
3937 for (auto &C : RepeatedSequenceLocs) {
3938
3939 if (C.isAvailableAcrossAndOutOfSeq(Reg: RISCV::X5, TRI: RegInfo)) {
3940 // X5 is available, just need the call
3941 unsigned CandCallOverhead = 8;
3942 C.setCallInfo(CID: MachineOutlinerDefault, CO: CandCallOverhead);
3943 } else {
3944 // X5 unavailable, need save + call + restore
3945 // Save (2-4) + Call (8) + Restore (2-4)
3946 unsigned CandCallOverhead = InstrSizeCExt + 8 + InstrSizeCExt;
3947 C.setCallInfo(CID: MachineOutlinerRegSave, CO: CandCallOverhead);
3948 }
3949 }
3950 } else {
3951 for (auto &C : RepeatedSequenceLocs)
3952 C.setCallInfo(CID: MOCI, CO: CallOverhead);
3953 }
3954
3955 unsigned SequenceSize = 0;
3956 for (auto &MI : Candidate)
3957 SequenceSize += getInstSizeInBytes(MI);
3958
3959 return std::make_unique<outliner::OutlinedFunction>(
3960 args&: RepeatedSequenceLocs, args&: SequenceSize, args&: FrameOverhead, args&: MOCI);
3961}
3962
3963outliner::InstrType
3964RISCVInstrInfo::getOutliningTypeImpl(const MachineModuleInfo &MMI,
3965 MachineBasicBlock::iterator &MBBI,
3966 unsigned Flags) const {
3967 MachineInstr &MI = *MBBI;
3968 MachineBasicBlock *MBB = MI.getParent();
3969 const TargetRegisterInfo *TRI =
3970 MBB->getParent()->getSubtarget().getRegisterInfo();
3971 const auto &F = MI.getMF()->getFunction();
3972
3973 // We can only outline CFI instructions if we will tail call the outlined
3974 // function, or fix up the CFI offsets. Currently, CFI instructions are
3975 // outlined only if in a tail call.
3976 if (MI.isCFIInstruction())
3977 return outliner::InstrType::Legal;
3978
3979 if (cannotInsertTailCall(MBB: *MBB) &&
3980 (MI.isReturn() || isMIModifiesReg(MI, TRI, RegNo: RISCV::X5)))
3981 return outliner::InstrType::Illegal;
3982
3983 // Make sure the operands don't reference something unsafe.
3984 for (const auto &MO : MI.operands()) {
3985
3986 // pcrel-hi and pcrel-lo can't put in separate sections, filter that out
3987 // if any possible.
3988 if (MO.getTargetFlags() == RISCVII::MO_PCREL_LO &&
3989 (MI.getMF()->getTarget().getFunctionSections() || F.hasComdat() ||
3990 F.hasSection() || F.getSectionPrefix()))
3991 return outliner::InstrType::Illegal;
3992 }
3993
3994 if (isLPAD(MI))
3995 return outliner::InstrType::Illegal;
3996
3997 return outliner::InstrType::Legal;
3998}
3999
4000void RISCVInstrInfo::buildOutlinedFrame(
4001 MachineBasicBlock &MBB, MachineFunction &MF,
4002 const outliner::OutlinedFunction &OF) const {
4003
4004 if (OF.FrameConstructionID == MachineOutlinerTailCall)
4005 return;
4006
4007 MBB.addLiveIn(PhysReg: RISCV::X5);
4008
4009 // Add in a return instruction to the end of the outlined frame.
4010 MBB.insert(I: MBB.end(), MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: RISCV::JALR), DestReg: RISCV::X0)
4011 .addReg(RegNo: RISCV::X5)
4012 .addImm(Val: 0));
4013}
4014
4015MachineBasicBlock::iterator RISCVInstrInfo::insertOutlinedCall(
4016 Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It,
4017 MachineFunction &MF, outliner::Candidate &C) const {
4018
4019 if (C.CallConstructionID == MachineOutlinerTailCall) {
4020 const RISCVMachineFunctionInfo *RVFI =
4021 MF.getInfo<RISCVMachineFunctionInfo>();
4022 unsigned TailOpc =
4023 RVFI->hasCFProtectionBranch() ? RISCV::PseudoTAILX7 : RISCV::PseudoTAIL;
4024 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: TailOpc))
4025 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName()),
4026 /*Offset=*/0, TargetFlags: RISCVII::MO_CALL));
4027 return It;
4028 }
4029
4030 if (C.CallConstructionID == MachineOutlinerRegSave) {
4031 Register SaveReg = findRegisterToSaveX5To(C, TRI: RegInfo);
4032 assert(SaveReg && "Cannot find an available register to save/restore X5.");
4033
4034 // Save: ADDI SaveReg, X5, 0 (equivalent to MV SaveReg, X5)
4035 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: RISCV::ADDI), DestReg: SaveReg)
4036 .addReg(RegNo: RISCV::X5)
4037 .addImm(Val: 0));
4038 It++;
4039
4040 // Call: PseudoCALLReg X5
4041 It = MBB.insert(
4042 I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: RISCV::PseudoCALLReg), DestReg: RISCV::X5)
4043 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName()), Offset: 0,
4044 TargetFlags: RISCVII::MO_CALL));
4045 MachineBasicBlock::iterator CallPt = It;
4046 It++;
4047
4048 // Restore: ADDI X5, SaveReg, 0 (equivalent to MV X5, SaveReg)
4049 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: RISCV::ADDI), DestReg: RISCV::X5)
4050 .addReg(RegNo: SaveReg)
4051 .addImm(Val: 0));
4052
4053 return CallPt;
4054 }
4055
4056 // Add in a call instruction to the outlined function at the given location.
4057 It = MBB.insert(I: It,
4058 MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: RISCV::PseudoCALLReg), DestReg: RISCV::X5)
4059 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName()), Offset: 0,
4060 TargetFlags: RISCVII::MO_CALL));
4061 return It;
4062}
4063
4064void RISCVInstrInfo::buildClearRegister(Register Reg, MachineBasicBlock &MBB,
4065 MachineBasicBlock::iterator Iter,
4066 DebugLoc &DL,
4067 bool AllowSideEffects) const {
4068
4069 const MachineFunction &MF = *MBB.getParent();
4070 const RISCVRegisterInfo &TRI = *STI.getRegisterInfo();
4071
4072 if (TRI.isGeneralPurposeRegister(MF, Reg)) {
4073 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: RISCV::PseudoClearGPR), DestReg: Reg);
4074 } else if (RISCV::FPR32RegClass.contains(Reg)) {
4075 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: RISCV::PseudoClearFPR32), DestReg: Reg);
4076 } else if (RISCV::FPR64RegClass.contains(Reg)) {
4077 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: RISCV::PseudoClearFPR64), DestReg: Reg);
4078 } else if (RISCV::FPR128RegClass.contains(Reg)) {
4079 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: RISCV::PseudoClearFPR128), DestReg: Reg);
4080 } else if (RISCV::VRRegClass.contains(Reg)) {
4081 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: RISCV::PseudoClearVR), DestReg: Reg);
4082 } else {
4083 llvm::reportFatalInternalError(
4084 reason: "buildClearRegister is not implemented for " + TRI.getRegAsmName(Reg));
4085 }
4086}
4087
4088std::optional<RegImmPair> RISCVInstrInfo::isAddImmediate(const MachineInstr &MI,
4089 Register Reg) const {
4090 // TODO: Handle cases where Reg is a super- or sub-register of the
4091 // destination register.
4092 const MachineOperand &Op0 = MI.getOperand(i: 0);
4093 if (!Op0.isReg() || Reg != Op0.getReg())
4094 return std::nullopt;
4095
4096 // Don't consider ADDIW as a candidate because the caller may not be aware
4097 // of its sign extension behaviour.
4098 if (MI.getOpcode() == RISCV::ADDI && MI.getOperand(i: 1).isReg() &&
4099 MI.getOperand(i: 2).isImm())
4100 return RegImmPair{MI.getOperand(i: 1).getReg(), MI.getOperand(i: 2).getImm()};
4101
4102 return std::nullopt;
4103}
4104
4105// MIR printer helper function to annotate Operands with a comment.
4106std::string RISCVInstrInfo::createMIROperandComment(const MachineInstr &MI,
4107 const MachineOperand &Op,
4108 unsigned OpIdx) const {
4109 // Print a generic comment for this operand if there is one.
4110 std::string GenericComment =
4111 TargetInstrInfo::createMIROperandComment(MI, Op, OpIdx);
4112 if (!GenericComment.empty())
4113 return GenericComment;
4114
4115 const MCInstrDesc &Desc = MI.getDesc();
4116 if (OpIdx >= Desc.getNumOperands())
4117 return std::string();
4118
4119 std::string Comment;
4120 raw_string_ostream OS(Comment);
4121
4122 const MCOperandInfo &OpInfo = Desc.operands()[OpIdx];
4123
4124 // Print the full VType operand of vsetvli/vsetivli instructions, and the SEW
4125 // operand of vector codegen pseudos.
4126 switch (OpInfo.OperandType) {
4127 case RISCVOp::OPERAND_VTYPEI10:
4128 case RISCVOp::OPERAND_VTYPEI11: {
4129 unsigned Imm = Op.getImm();
4130 RISCVVType::printVType(VType: Imm, OS);
4131 break;
4132 }
4133 case RISCVOp::OPERAND_XSFMM_VTYPE: {
4134 unsigned Imm = Op.getImm();
4135 RISCVVType::printXSfmmVType(VType: Imm, OS);
4136 break;
4137 }
4138 case RISCVOp::OPERAND_XSFMM_TWIDEN: {
4139 unsigned Imm = Op.getImm();
4140 OS << "w" << Imm;
4141 break;
4142 }
4143 case RISCVOp::OPERAND_SEW:
4144 case RISCVOp::OPERAND_SEW_MASK: {
4145 unsigned Log2SEW = Op.getImm();
4146 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
4147 assert(RISCVVType::isValidSEW(SEW) && "Unexpected SEW");
4148 OS << "e" << SEW;
4149 break;
4150 }
4151 case RISCVOp::OPERAND_VEC_POLICY: {
4152 unsigned Policy = Op.getImm();
4153 assert(Policy <= (RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC) &&
4154 "Invalid Policy Value");
4155 OS << (Policy & RISCVVType::TAIL_AGNOSTIC ? "ta" : "tu") << ", "
4156 << (Policy & RISCVVType::MASK_AGNOSTIC ? "ma" : "mu");
4157 break;
4158 }
4159 case RISCVOp::OPERAND_AVL:
4160 if (Op.isImm() && Op.getImm() == -1)
4161 OS << "vl=VLMAX";
4162 else
4163 OS << "vl";
4164 break;
4165 case RISCVOp::OPERAND_VEC_RM:
4166 if (RISCVII::usesVXRM(TSFlags: Desc.TSFlags)) {
4167 assert(RISCVVXRndMode::isValidRoundingMode(Op.getImm()));
4168 auto VXRM = static_cast<RISCVVXRndMode::RoundingMode>(Op.getImm());
4169 OS << "vxrm=" << RISCVVXRndMode::roundingModeToString(RndMode: VXRM);
4170 } else {
4171 assert(RISCVFPRndMode::isValidRoundingMode(Op.getImm()));
4172 auto FRM = static_cast<RISCVFPRndMode::RoundingMode>(Op.getImm());
4173 OS << "frm=" << RISCVFPRndMode::roundingModeToString(RndMode: FRM);
4174 }
4175 break;
4176 }
4177
4178 return Comment;
4179}
4180
4181// clang-format off
4182#define CASE_RVV_OPCODE_UNMASK_LMUL(OP, LMUL) \
4183 RISCV::Pseudo##OP##_##LMUL
4184
4185#define CASE_RVV_OPCODE_MASK_LMUL(OP, LMUL) \
4186 RISCV::Pseudo##OP##_##LMUL##_MASK
4187
4188#define CASE_RVV_OPCODE_LMUL(OP, LMUL) \
4189 CASE_RVV_OPCODE_UNMASK_LMUL(OP, LMUL): \
4190 case CASE_RVV_OPCODE_MASK_LMUL(OP, LMUL)
4191
4192#define CASE_RVV_OPCODE_UNMASK_WIDEN(OP) \
4193 CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF8): \
4194 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF4): \
4195 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF2): \
4196 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M1): \
4197 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M2): \
4198 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M4)
4199
4200#define CASE_RVV_OPCODE_UNMASK(OP) \
4201 CASE_RVV_OPCODE_UNMASK_WIDEN(OP): \
4202 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M8)
4203
4204#define CASE_RVV_OPCODE_MASK_WIDEN(OP) \
4205 CASE_RVV_OPCODE_MASK_LMUL(OP, MF8): \
4206 case CASE_RVV_OPCODE_MASK_LMUL(OP, MF4): \
4207 case CASE_RVV_OPCODE_MASK_LMUL(OP, MF2): \
4208 case CASE_RVV_OPCODE_MASK_LMUL(OP, M1): \
4209 case CASE_RVV_OPCODE_MASK_LMUL(OP, M2): \
4210 case CASE_RVV_OPCODE_MASK_LMUL(OP, M4)
4211
4212#define CASE_RVV_OPCODE_MASK(OP) \
4213 CASE_RVV_OPCODE_MASK_WIDEN(OP): \
4214 case CASE_RVV_OPCODE_MASK_LMUL(OP, M8)
4215
4216#define CASE_RVV_OPCODE_WIDEN(OP) \
4217 CASE_RVV_OPCODE_UNMASK_WIDEN(OP): \
4218 case CASE_RVV_OPCODE_MASK_WIDEN(OP)
4219
4220#define CASE_RVV_OPCODE(OP) \
4221 CASE_RVV_OPCODE_UNMASK(OP): \
4222 case CASE_RVV_OPCODE_MASK(OP)
4223// clang-format on
4224
4225// clang-format off
4226#define CASE_VMA_OPCODE_COMMON(OP, TYPE, LMUL) \
4227 RISCV::PseudoV##OP##_##TYPE##_##LMUL
4228
4229#define CASE_VMA_OPCODE_LMULS(OP, TYPE) \
4230 CASE_VMA_OPCODE_COMMON(OP, TYPE, MF8): \
4231 case CASE_VMA_OPCODE_COMMON(OP, TYPE, MF4): \
4232 case CASE_VMA_OPCODE_COMMON(OP, TYPE, MF2): \
4233 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M1): \
4234 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M2): \
4235 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M4): \
4236 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M8)
4237
4238// VFMA instructions are SEW specific.
4239#define CASE_VFMA_OPCODE_COMMON(OP, TYPE, LMUL, SEW) \
4240 RISCV::PseudoV##OP##_##TYPE##_##LMUL##_##SEW
4241
4242#define CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE, SEW) \
4243 CASE_VFMA_OPCODE_COMMON(OP, TYPE, M1, SEW): \
4244 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M2, SEW): \
4245 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M4, SEW): \
4246 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M8, SEW)
4247
4248#define CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE, SEW) \
4249 CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF2, SEW): \
4250 case CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE, SEW)
4251
4252#define CASE_VFMA_OPCODE_LMULS_MF4(OP, TYPE, SEW) \
4253 CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF4, SEW): \
4254 case CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE, SEW)
4255
4256#define CASE_VFMA_OPCODE_VV(OP) \
4257 CASE_VFMA_OPCODE_LMULS_MF4(OP, VV, E16): \
4258 case CASE_VFMA_OPCODE_LMULS_MF4(OP##_ALT, VV, E16): \
4259 case CASE_VFMA_OPCODE_LMULS_MF2(OP, VV, E32): \
4260 case CASE_VFMA_OPCODE_LMULS_M1(OP, VV, E64)
4261
4262#define CASE_VFMA_SPLATS(OP) \
4263 CASE_VFMA_OPCODE_LMULS_MF4(OP, VFPR16, E16): \
4264 case CASE_VFMA_OPCODE_LMULS_MF4(OP##_ALT, VFPR16, E16): \
4265 case CASE_VFMA_OPCODE_LMULS_MF2(OP, VFPR32, E32): \
4266 case CASE_VFMA_OPCODE_LMULS_M1(OP, VFPR64, E64)
4267// clang-format on
4268
4269bool RISCVInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
4270 unsigned &SrcOpIdx1,
4271 unsigned &SrcOpIdx2) const {
4272 const MCInstrDesc &Desc = MI.getDesc();
4273 if (!Desc.isCommutable())
4274 return false;
4275
4276 switch (MI.getOpcode()) {
4277 case RISCV::TH_MVEQZ:
4278 case RISCV::TH_MVNEZ:
4279 // We can't commute operands if operand 2 (i.e., rs1 in
4280 // mveqz/mvnez rd,rs1,rs2) is the zero-register (as it is
4281 // not valid as the in/out-operand 1).
4282 if (MI.getOperand(i: 2).getReg() == RISCV::X0)
4283 return false;
4284 // Operands 1 and 2 are commutable, if we switch the opcode.
4285 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 1, CommutableOpIdx2: 2);
4286 case RISCV::QC_SELECTIEQ:
4287 case RISCV::QC_SELECTINE:
4288 case RISCV::QC_SELECTIIEQ:
4289 case RISCV::QC_SELECTIINE:
4290 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 1, CommutableOpIdx2: 2);
4291 case RISCV::QC_MVEQ:
4292 case RISCV::QC_MVNE:
4293 case RISCV::QC_MVLT:
4294 case RISCV::QC_MVGE:
4295 case RISCV::QC_MVLTU:
4296 case RISCV::QC_MVGEU:
4297 case RISCV::QC_MVEQI:
4298 case RISCV::QC_MVNEI:
4299 case RISCV::QC_MVLTI:
4300 case RISCV::QC_MVGEI:
4301 case RISCV::QC_MVLTUI:
4302 case RISCV::QC_MVGEUI:
4303 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 1, CommutableOpIdx2: 4);
4304 case RISCV::TH_MULA:
4305 case RISCV::TH_MULAW:
4306 case RISCV::TH_MULAH:
4307 case RISCV::TH_MULS:
4308 case RISCV::TH_MULSW:
4309 case RISCV::TH_MULSH:
4310 // Operands 2 and 3 are commutable.
4311 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 2, CommutableOpIdx2: 3);
4312 case RISCV::PseudoCCMOVGPRNoX0:
4313 case RISCV::PseudoCCMOVGPR:
4314 // Operands 1 and 2 are commutable.
4315 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 1, CommutableOpIdx2: 2);
4316 case CASE_RVV_OPCODE(VADD_VV):
4317 case CASE_RVV_OPCODE(VAND_VV):
4318 case CASE_RVV_OPCODE(VOR_VV):
4319 case CASE_RVV_OPCODE(VXOR_VV):
4320 case CASE_RVV_OPCODE_MASK(VMSEQ_VV):
4321 case CASE_RVV_OPCODE_MASK(VMSNE_VV):
4322 case CASE_RVV_OPCODE(VMIN_VV):
4323 case CASE_RVV_OPCODE(VMINU_VV):
4324 case CASE_RVV_OPCODE(VMAX_VV):
4325 case CASE_RVV_OPCODE(VMAXU_VV):
4326 case CASE_RVV_OPCODE(VMUL_VV):
4327 case CASE_RVV_OPCODE(VMULH_VV):
4328 case CASE_RVV_OPCODE(VMULHU_VV):
4329 case CASE_RVV_OPCODE_WIDEN(VWADD_VV):
4330 case CASE_RVV_OPCODE_WIDEN(VWADDU_VV):
4331 case CASE_RVV_OPCODE_WIDEN(VWMUL_VV):
4332 case CASE_RVV_OPCODE_WIDEN(VWMULU_VV):
4333 case CASE_RVV_OPCODE_WIDEN(VWMACC_VV):
4334 case CASE_RVV_OPCODE_WIDEN(VWMACCU_VV):
4335 case CASE_RVV_OPCODE(VABD_VV):
4336 case CASE_RVV_OPCODE(VABDU_VV):
4337 case CASE_RVV_OPCODE_WIDEN(VWABDA_VV):
4338 case CASE_RVV_OPCODE_WIDEN(VWABDAU_VV):
4339 case CASE_RVV_OPCODE_UNMASK(VADC_VVM):
4340 case CASE_RVV_OPCODE(VSADD_VV):
4341 case CASE_RVV_OPCODE(VSADDU_VV):
4342 case CASE_RVV_OPCODE(VAADD_VV):
4343 case CASE_RVV_OPCODE(VAADDU_VV):
4344 case CASE_RVV_OPCODE(VSMUL_VV):
4345 case CASE_RVV_OPCODE_LMUL(VDOT4A_VV, MF2):
4346 case CASE_RVV_OPCODE_LMUL(VDOT4A_VV, M1):
4347 case CASE_RVV_OPCODE_LMUL(VDOT4A_VV, M2):
4348 case CASE_RVV_OPCODE_LMUL(VDOT4A_VV, M4):
4349 case CASE_RVV_OPCODE_LMUL(VDOT4A_VV, M8):
4350 case CASE_RVV_OPCODE_LMUL(VDOT4AU_VV, MF2):
4351 case CASE_RVV_OPCODE_LMUL(VDOT4AU_VV, M1):
4352 case CASE_RVV_OPCODE_LMUL(VDOT4AU_VV, M2):
4353 case CASE_RVV_OPCODE_LMUL(VDOT4AU_VV, M4):
4354 case CASE_RVV_OPCODE_LMUL(VDOT4AU_VV, M8):
4355 // Operands 2 and 3 are commutable.
4356 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 2, CommutableOpIdx2: 3);
4357 case CASE_VFMA_SPLATS(FMADD):
4358 case CASE_VFMA_SPLATS(FMSUB):
4359 case CASE_VFMA_SPLATS(FMACC):
4360 case CASE_VFMA_SPLATS(FMSAC):
4361 case CASE_VFMA_SPLATS(FNMADD):
4362 case CASE_VFMA_SPLATS(FNMSUB):
4363 case CASE_VFMA_SPLATS(FNMACC):
4364 case CASE_VFMA_SPLATS(FNMSAC):
4365 case CASE_VFMA_OPCODE_VV(FMACC):
4366 case CASE_VFMA_OPCODE_VV(FMSAC):
4367 case CASE_VFMA_OPCODE_VV(FNMACC):
4368 case CASE_VFMA_OPCODE_VV(FNMSAC):
4369 case CASE_VMA_OPCODE_LMULS(MADD, VX):
4370 case CASE_VMA_OPCODE_LMULS(NMSUB, VX):
4371 case CASE_VMA_OPCODE_LMULS(MACC, VX):
4372 case CASE_VMA_OPCODE_LMULS(NMSAC, VX):
4373 case CASE_VMA_OPCODE_LMULS(MACC, VV):
4374 case CASE_VMA_OPCODE_LMULS(NMSAC, VV): {
4375 // If the tail policy is undisturbed we can't commute.
4376 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags));
4377 if ((MI.getOperand(i: RISCVII::getVecPolicyOpNum(Desc: MI.getDesc())).getImm() &
4378 1) == 0)
4379 return false;
4380
4381 // For these instructions we can only swap operand 1 and operand 3 by
4382 // changing the opcode.
4383 unsigned CommutableOpIdx1 = 1;
4384 unsigned CommutableOpIdx2 = 3;
4385 if (!fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1,
4386 CommutableOpIdx2))
4387 return false;
4388 return true;
4389 }
4390 case CASE_VFMA_OPCODE_VV(FMADD):
4391 case CASE_VFMA_OPCODE_VV(FMSUB):
4392 case CASE_VFMA_OPCODE_VV(FNMADD):
4393 case CASE_VFMA_OPCODE_VV(FNMSUB):
4394 case CASE_VMA_OPCODE_LMULS(MADD, VV):
4395 case CASE_VMA_OPCODE_LMULS(NMSUB, VV): {
4396 // If the tail policy is undisturbed we can't commute.
4397 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags));
4398 if ((MI.getOperand(i: RISCVII::getVecPolicyOpNum(Desc: MI.getDesc())).getImm() &
4399 1) == 0)
4400 return false;
4401
4402 // For these instructions we have more freedom. We can commute with the
4403 // other multiplicand or with the addend/subtrahend/minuend.
4404
4405 // Any fixed operand must be from source 1, 2 or 3.
4406 if (SrcOpIdx1 != CommuteAnyOperandIndex && SrcOpIdx1 > 3)
4407 return false;
4408 if (SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx2 > 3)
4409 return false;
4410
4411 // It both ops are fixed one must be the tied source.
4412 if (SrcOpIdx1 != CommuteAnyOperandIndex &&
4413 SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx1 != 1 && SrcOpIdx2 != 1)
4414 return false;
4415
4416 // Look for two different register operands assumed to be commutable
4417 // regardless of the FMA opcode. The FMA opcode is adjusted later if
4418 // needed.
4419 if (SrcOpIdx1 == CommuteAnyOperandIndex ||
4420 SrcOpIdx2 == CommuteAnyOperandIndex) {
4421 // At least one of operands to be commuted is not specified and
4422 // this method is free to choose appropriate commutable operands.
4423 unsigned CommutableOpIdx1 = SrcOpIdx1;
4424 if (SrcOpIdx1 == SrcOpIdx2) {
4425 // Both of operands are not fixed. Set one of commutable
4426 // operands to the tied source.
4427 CommutableOpIdx1 = 1;
4428 } else if (SrcOpIdx1 == CommuteAnyOperandIndex) {
4429 // Only one of the operands is not fixed.
4430 CommutableOpIdx1 = SrcOpIdx2;
4431 }
4432
4433 // CommutableOpIdx1 is well defined now. Let's choose another commutable
4434 // operand and assign its index to CommutableOpIdx2.
4435 unsigned CommutableOpIdx2;
4436 if (CommutableOpIdx1 != 1) {
4437 // If we haven't already used the tied source, we must use it now.
4438 CommutableOpIdx2 = 1;
4439 } else {
4440 Register Op1Reg = MI.getOperand(i: CommutableOpIdx1).getReg();
4441
4442 // The commuted operands should have different registers.
4443 // Otherwise, the commute transformation does not change anything and
4444 // is useless. We use this as a hint to make our decision.
4445 if (Op1Reg != MI.getOperand(i: 2).getReg())
4446 CommutableOpIdx2 = 2;
4447 else
4448 CommutableOpIdx2 = 3;
4449 }
4450
4451 // Assign the found pair of commutable indices to SrcOpIdx1 and
4452 // SrcOpIdx2 to return those values.
4453 if (!fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1,
4454 CommutableOpIdx2))
4455 return false;
4456 }
4457
4458 return true;
4459 }
4460 }
4461
4462 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
4463}
4464
4465// clang-format off
4466#define CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL) \
4467 case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL: \
4468 Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL; \
4469 break;
4470
4471#define CASE_VMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE) \
4472 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF8) \
4473 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4) \
4474 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2) \
4475 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1) \
4476 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2) \
4477 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4) \
4478 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8)
4479
4480// VFMA depends on SEW.
4481#define CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL, SEW) \
4482 case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL##_##SEW: \
4483 Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL##_##SEW; \
4484 break;
4485
4486#define CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE, SEW) \
4487 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1, SEW) \
4488 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2, SEW) \
4489 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4, SEW) \
4490 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8, SEW)
4491
4492#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE, SEW) \
4493 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2, SEW) \
4494 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE, SEW)
4495
4496#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE, SEW) \
4497 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4, SEW) \
4498 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE, SEW)
4499
4500#define CASE_VFMA_CHANGE_OPCODE_VV(OLDOP, NEWOP) \
4501 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VV, E16) \
4502 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP##_ALT, NEWOP##_ALT, VV, E16) \
4503 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VV, E32) \
4504 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VV, E64)
4505
4506#define CASE_VFMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP) \
4507 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VFPR16, E16) \
4508 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP##_ALT, NEWOP##_ALT, VFPR16, E16) \
4509 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VFPR32, E32) \
4510 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VFPR64, E64)
4511// clang-format on
4512
4513MachineInstr *RISCVInstrInfo::commuteInstructionImpl(MachineInstr &MI,
4514 bool NewMI,
4515 unsigned OpIdx1,
4516 unsigned OpIdx2) const {
4517 auto cloneIfNew = [NewMI](MachineInstr &MI) -> MachineInstr & {
4518 if (NewMI)
4519 return *MI.getParent()->getParent()->CloneMachineInstr(Orig: &MI);
4520 return MI;
4521 };
4522
4523 switch (MI.getOpcode()) {
4524 case RISCV::TH_MVEQZ:
4525 case RISCV::TH_MVNEZ: {
4526 auto &WorkingMI = cloneIfNew(MI);
4527 WorkingMI.setDesc(get(Opcode: MI.getOpcode() == RISCV::TH_MVEQZ ? RISCV::TH_MVNEZ
4528 : RISCV::TH_MVEQZ));
4529 return TargetInstrInfo::commuteInstructionImpl(MI&: WorkingMI, NewMI: false, OpIdx1,
4530 OpIdx2);
4531 }
4532 case RISCV::QC_SELECTIEQ:
4533 case RISCV::QC_SELECTINE:
4534 case RISCV::QC_SELECTIIEQ:
4535 case RISCV::QC_SELECTIINE:
4536 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
4537 case RISCV::QC_MVEQ:
4538 case RISCV::QC_MVNE:
4539 case RISCV::QC_MVLT:
4540 case RISCV::QC_MVGE:
4541 case RISCV::QC_MVLTU:
4542 case RISCV::QC_MVGEU:
4543 case RISCV::QC_MVEQI:
4544 case RISCV::QC_MVNEI:
4545 case RISCV::QC_MVLTI:
4546 case RISCV::QC_MVGEI:
4547 case RISCV::QC_MVLTUI:
4548 case RISCV::QC_MVGEUI: {
4549 auto &WorkingMI = cloneIfNew(MI);
4550 WorkingMI.setDesc(get(Opcode: getInverseXqcicmOpcode(Opcode: MI.getOpcode())));
4551 return TargetInstrInfo::commuteInstructionImpl(MI&: WorkingMI, NewMI: false, OpIdx1,
4552 OpIdx2);
4553 }
4554 case RISCV::PseudoCCMOVGPRNoX0:
4555 case RISCV::PseudoCCMOVGPR: {
4556 // CCMOV can be commuted by inverting the condition.
4557 unsigned BCC = MI.getOperand(i: MI.getNumExplicitOperands() - 3).getImm();
4558 BCC = RISCVCC::getInverseBranchOpcode(BCC);
4559 auto &WorkingMI = cloneIfNew(MI);
4560 WorkingMI.getOperand(i: MI.getNumExplicitOperands() - 3).setImm(BCC);
4561 return TargetInstrInfo::commuteInstructionImpl(MI&: WorkingMI, /*NewMI*/ false,
4562 OpIdx1, OpIdx2);
4563 }
4564 case CASE_VFMA_SPLATS(FMACC):
4565 case CASE_VFMA_SPLATS(FMADD):
4566 case CASE_VFMA_SPLATS(FMSAC):
4567 case CASE_VFMA_SPLATS(FMSUB):
4568 case CASE_VFMA_SPLATS(FNMACC):
4569 case CASE_VFMA_SPLATS(FNMADD):
4570 case CASE_VFMA_SPLATS(FNMSAC):
4571 case CASE_VFMA_SPLATS(FNMSUB):
4572 case CASE_VFMA_OPCODE_VV(FMACC):
4573 case CASE_VFMA_OPCODE_VV(FMSAC):
4574 case CASE_VFMA_OPCODE_VV(FNMACC):
4575 case CASE_VFMA_OPCODE_VV(FNMSAC):
4576 case CASE_VMA_OPCODE_LMULS(MADD, VX):
4577 case CASE_VMA_OPCODE_LMULS(NMSUB, VX):
4578 case CASE_VMA_OPCODE_LMULS(MACC, VX):
4579 case CASE_VMA_OPCODE_LMULS(NMSAC, VX):
4580 case CASE_VMA_OPCODE_LMULS(MACC, VV):
4581 case CASE_VMA_OPCODE_LMULS(NMSAC, VV): {
4582 // It only make sense to toggle these between clobbering the
4583 // addend/subtrahend/minuend one of the multiplicands.
4584 assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");
4585 assert((OpIdx1 == 3 || OpIdx2 == 3) && "Unexpected opcode index");
4586 unsigned Opc;
4587 switch (MI.getOpcode()) {
4588 default:
4589 llvm_unreachable("Unexpected opcode");
4590 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMACC, FMADD)
4591 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMADD, FMACC)
4592 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMSAC, FMSUB)
4593 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMSUB, FMSAC)
4594 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMACC, FNMADD)
4595 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMADD, FNMACC)
4596 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMSAC, FNMSUB)
4597 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMSUB, FNMSAC)
4598 CASE_VFMA_CHANGE_OPCODE_VV(FMACC, FMADD)
4599 CASE_VFMA_CHANGE_OPCODE_VV(FMSAC, FMSUB)
4600 CASE_VFMA_CHANGE_OPCODE_VV(FNMACC, FNMADD)
4601 CASE_VFMA_CHANGE_OPCODE_VV(FNMSAC, FNMSUB)
4602 CASE_VMA_CHANGE_OPCODE_LMULS(MACC, MADD, VX)
4603 CASE_VMA_CHANGE_OPCODE_LMULS(MADD, MACC, VX)
4604 CASE_VMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VX)
4605 CASE_VMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VX)
4606 CASE_VMA_CHANGE_OPCODE_LMULS(MACC, MADD, VV)
4607 CASE_VMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VV)
4608 }
4609
4610 auto &WorkingMI = cloneIfNew(MI);
4611 WorkingMI.setDesc(get(Opcode: Opc));
4612 return TargetInstrInfo::commuteInstructionImpl(MI&: WorkingMI, /*NewMI=*/false,
4613 OpIdx1, OpIdx2);
4614 }
4615 case CASE_VFMA_OPCODE_VV(FMADD):
4616 case CASE_VFMA_OPCODE_VV(FMSUB):
4617 case CASE_VFMA_OPCODE_VV(FNMADD):
4618 case CASE_VFMA_OPCODE_VV(FNMSUB):
4619 case CASE_VMA_OPCODE_LMULS(MADD, VV):
4620 case CASE_VMA_OPCODE_LMULS(NMSUB, VV): {
4621 assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");
4622 // If one of the operands, is the addend we need to change opcode.
4623 // Otherwise we're just swapping 2 of the multiplicands.
4624 if (OpIdx1 == 3 || OpIdx2 == 3) {
4625 unsigned Opc;
4626 switch (MI.getOpcode()) {
4627 default:
4628 llvm_unreachable("Unexpected opcode");
4629 CASE_VFMA_CHANGE_OPCODE_VV(FMADD, FMACC)
4630 CASE_VFMA_CHANGE_OPCODE_VV(FMSUB, FMSAC)
4631 CASE_VFMA_CHANGE_OPCODE_VV(FNMADD, FNMACC)
4632 CASE_VFMA_CHANGE_OPCODE_VV(FNMSUB, FNMSAC)
4633 CASE_VMA_CHANGE_OPCODE_LMULS(MADD, MACC, VV)
4634 CASE_VMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VV)
4635 }
4636
4637 auto &WorkingMI = cloneIfNew(MI);
4638 WorkingMI.setDesc(get(Opcode: Opc));
4639 return TargetInstrInfo::commuteInstructionImpl(MI&: WorkingMI, /*NewMI=*/false,
4640 OpIdx1, OpIdx2);
4641 }
4642 // Let the default code handle it.
4643 break;
4644 }
4645 }
4646
4647 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
4648}
4649
4650#undef CASE_VMA_CHANGE_OPCODE_COMMON
4651#undef CASE_VMA_CHANGE_OPCODE_LMULS
4652#undef CASE_VFMA_CHANGE_OPCODE_COMMON
4653#undef CASE_VFMA_CHANGE_OPCODE_LMULS_M1
4654#undef CASE_VFMA_CHANGE_OPCODE_LMULS_MF2
4655#undef CASE_VFMA_CHANGE_OPCODE_LMULS_MF4
4656#undef CASE_VFMA_CHANGE_OPCODE_VV
4657#undef CASE_VFMA_CHANGE_OPCODE_SPLATS
4658
4659#undef CASE_RVV_OPCODE_UNMASK_LMUL
4660#undef CASE_RVV_OPCODE_MASK_LMUL
4661#undef CASE_RVV_OPCODE_LMUL
4662#undef CASE_RVV_OPCODE_UNMASK_WIDEN
4663#undef CASE_RVV_OPCODE_UNMASK
4664#undef CASE_RVV_OPCODE_MASK_WIDEN
4665#undef CASE_RVV_OPCODE_MASK
4666#undef CASE_RVV_OPCODE_WIDEN
4667#undef CASE_RVV_OPCODE
4668
4669#undef CASE_VMA_OPCODE_COMMON
4670#undef CASE_VMA_OPCODE_LMULS
4671#undef CASE_VFMA_OPCODE_COMMON
4672#undef CASE_VFMA_OPCODE_LMULS_M1
4673#undef CASE_VFMA_OPCODE_LMULS_MF2
4674#undef CASE_VFMA_OPCODE_LMULS_MF4
4675#undef CASE_VFMA_OPCODE_VV
4676#undef CASE_VFMA_SPLATS
4677
4678bool RISCVInstrInfo::simplifyInstruction(MachineInstr &MI) const {
4679 switch (MI.getOpcode()) {
4680 default:
4681 break;
4682 case RISCV::ADD:
4683 case RISCV::OR:
4684 case RISCV::XOR:
4685 // Normalize (so we hit the next if clause).
4686 // add/[x]or rd, zero, rs => add/[x]or rd, rs, zero
4687 if (MI.getOperand(i: 1).getReg() == RISCV::X0)
4688 commuteInstruction(MI);
4689 // add/[x]or rd, rs, zero => addi rd, rs, 0
4690 if (MI.getOperand(i: 2).getReg() == RISCV::X0) {
4691 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4692 MI.setDesc(get(Opcode: RISCV::ADDI));
4693 return true;
4694 }
4695 // xor rd, rs, rs => addi rd, zero, 0
4696 if (MI.getOpcode() == RISCV::XOR &&
4697 MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg()) {
4698 MI.getOperand(i: 1).setReg(RISCV::X0);
4699 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4700 MI.setDesc(get(Opcode: RISCV::ADDI));
4701 return true;
4702 }
4703 break;
4704 case RISCV::ORI:
4705 case RISCV::XORI:
4706 // [x]ori rd, zero, N => addi rd, zero, N
4707 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4708 MI.setDesc(get(Opcode: RISCV::ADDI));
4709 return true;
4710 }
4711 break;
4712 case RISCV::SUB:
4713 // sub rd, rs, zero => addi rd, rs, 0
4714 if (MI.getOperand(i: 2).getReg() == RISCV::X0) {
4715 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4716 MI.setDesc(get(Opcode: RISCV::ADDI));
4717 return true;
4718 }
4719 break;
4720 case RISCV::SUBW:
4721 // subw rd, rs, zero => addiw rd, rs, 0
4722 if (MI.getOperand(i: 2).getReg() == RISCV::X0) {
4723 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4724 MI.setDesc(get(Opcode: RISCV::ADDIW));
4725 return true;
4726 }
4727 break;
4728 case RISCV::ADDW:
4729 // Normalize (so we hit the next if clause).
4730 // addw rd, zero, rs => addw rd, rs, zero
4731 if (MI.getOperand(i: 1).getReg() == RISCV::X0)
4732 commuteInstruction(MI);
4733 // addw rd, rs, zero => addiw rd, rs, 0
4734 if (MI.getOperand(i: 2).getReg() == RISCV::X0) {
4735 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4736 MI.setDesc(get(Opcode: RISCV::ADDIW));
4737 return true;
4738 }
4739 break;
4740 case RISCV::SH1ADD:
4741 case RISCV::SH1ADD_UW:
4742 case RISCV::SH2ADD:
4743 case RISCV::SH2ADD_UW:
4744 case RISCV::SH3ADD:
4745 case RISCV::SH3ADD_UW:
4746 // shNadd[.uw] rd, zero, rs => addi rd, rs, 0
4747 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4748 MI.removeOperand(OpNo: 1);
4749 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
4750 MI.setDesc(get(Opcode: RISCV::ADDI));
4751 return true;
4752 }
4753 // shNadd[.uw] rd, rs, zero => slli[.uw] rd, rs, N
4754 if (MI.getOperand(i: 2).getReg() == RISCV::X0) {
4755 MI.removeOperand(OpNo: 2);
4756 unsigned Opc = MI.getOpcode();
4757 if (Opc == RISCV::SH1ADD_UW || Opc == RISCV::SH2ADD_UW ||
4758 Opc == RISCV::SH3ADD_UW) {
4759 MI.addOperand(Op: MachineOperand::CreateImm(Val: getSHXADDUWShiftAmount(Opc)));
4760 MI.setDesc(get(Opcode: RISCV::SLLI_UW));
4761 return true;
4762 }
4763 MI.addOperand(Op: MachineOperand::CreateImm(Val: getSHXADDShiftAmount(Opc)));
4764 MI.setDesc(get(Opcode: RISCV::SLLI));
4765 return true;
4766 }
4767 break;
4768 case RISCV::AND:
4769 case RISCV::MUL:
4770 case RISCV::MULH:
4771 case RISCV::MULHSU:
4772 case RISCV::MULHU:
4773 case RISCV::MULW:
4774 // and rd, zero, rs => addi rd, zero, 0
4775 // mul* rd, zero, rs => addi rd, zero, 0
4776 // and rd, rs, zero => addi rd, zero, 0
4777 // mul* rd, rs, zero => addi rd, zero, 0
4778 if (MI.getOperand(i: 1).getReg() == RISCV::X0 ||
4779 MI.getOperand(i: 2).getReg() == RISCV::X0) {
4780 MI.getOperand(i: 1).setReg(RISCV::X0);
4781 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4782 MI.setDesc(get(Opcode: RISCV::ADDI));
4783 return true;
4784 }
4785 break;
4786 case RISCV::ANDI:
4787 // andi rd, zero, C => addi rd, zero, 0
4788 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4789 MI.getOperand(i: 2).setImm(0);
4790 MI.setDesc(get(Opcode: RISCV::ADDI));
4791 return true;
4792 }
4793 break;
4794 case RISCV::SLL:
4795 case RISCV::SRL:
4796 case RISCV::SRA:
4797 // shift rd, zero, rs => addi rd, zero, 0
4798 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4799 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4800 MI.setDesc(get(Opcode: RISCV::ADDI));
4801 return true;
4802 }
4803 // shift rd, rs, zero => addi rd, rs, 0
4804 if (MI.getOperand(i: 2).getReg() == RISCV::X0) {
4805 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4806 MI.setDesc(get(Opcode: RISCV::ADDI));
4807 return true;
4808 }
4809 break;
4810 case RISCV::SLLW:
4811 case RISCV::SRLW:
4812 case RISCV::SRAW:
4813 // shiftw rd, zero, rs => addi rd, zero, 0
4814 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4815 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4816 MI.setDesc(get(Opcode: RISCV::ADDI));
4817 return true;
4818 }
4819 break;
4820 case RISCV::SLLI:
4821 case RISCV::SRLI:
4822 case RISCV::SRAI:
4823 case RISCV::SLLIW:
4824 case RISCV::SRLIW:
4825 case RISCV::SRAIW:
4826 case RISCV::SLLI_UW:
4827 // shiftimm rd, zero, N => addi rd, zero, 0
4828 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4829 MI.getOperand(i: 2).setImm(0);
4830 MI.setDesc(get(Opcode: RISCV::ADDI));
4831 return true;
4832 }
4833 break;
4834 case RISCV::SLTU:
4835 case RISCV::ADD_UW:
4836 // sltu rd, zero, zero => addi rd, zero, 0
4837 // add.uw rd, zero, zero => addi rd, zero, 0
4838 if (MI.getOperand(i: 1).getReg() == RISCV::X0 &&
4839 MI.getOperand(i: 2).getReg() == RISCV::X0) {
4840 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4841 MI.setDesc(get(Opcode: RISCV::ADDI));
4842 return true;
4843 }
4844 // add.uw rd, zero, rs => addi rd, rs, 0
4845 if (MI.getOpcode() == RISCV::ADD_UW &&
4846 MI.getOperand(i: 1).getReg() == RISCV::X0) {
4847 MI.removeOperand(OpNo: 1);
4848 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
4849 MI.setDesc(get(Opcode: RISCV::ADDI));
4850 }
4851 break;
4852 case RISCV::SLTIU:
4853 // sltiu rd, zero, NZC => addi rd, zero, 1
4854 // sltiu rd, zero, 0 => addi rd, zero, 0
4855 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4856 MI.getOperand(i: 2).setImm(MI.getOperand(i: 2).getImm() != 0);
4857 MI.setDesc(get(Opcode: RISCV::ADDI));
4858 return true;
4859 }
4860 break;
4861 case RISCV::SEXT_H:
4862 case RISCV::SEXT_B:
4863 case RISCV::ZEXT_H_RV32:
4864 case RISCV::ZEXT_H_RV64:
4865 // sext.[hb] rd, zero => addi rd, zero, 0
4866 // zext.h rd, zero => addi rd, zero, 0
4867 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4868 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
4869 MI.setDesc(get(Opcode: RISCV::ADDI));
4870 return true;
4871 }
4872 break;
4873 case RISCV::MIN:
4874 case RISCV::MINU:
4875 case RISCV::MAX:
4876 case RISCV::MAXU:
4877 // min|max rd, rs, rs => addi rd, rs, 0
4878 if (MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg()) {
4879 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4880 MI.setDesc(get(Opcode: RISCV::ADDI));
4881 return true;
4882 }
4883 break;
4884 case RISCV::BEQ:
4885 case RISCV::BNE:
4886 // b{eq,ne} zero, rs, imm => b{eq,ne} rs, zero, imm
4887 if (MI.getOperand(i: 0).getReg() == RISCV::X0) {
4888 MachineOperand MO0 = MI.getOperand(i: 0);
4889 MI.removeOperand(OpNo: 0);
4890 MI.insert(InsertBefore: MI.operands_begin() + 1, Ops: {MO0});
4891 }
4892 break;
4893 case RISCV::BLTU:
4894 // bltu zero, rs, imm => bne rs, zero, imm
4895 if (MI.getOperand(i: 0).getReg() == RISCV::X0) {
4896 MachineOperand MO0 = MI.getOperand(i: 0);
4897 MI.removeOperand(OpNo: 0);
4898 MI.insert(InsertBefore: MI.operands_begin() + 1, Ops: {MO0});
4899 MI.setDesc(get(Opcode: RISCV::BNE));
4900 }
4901 break;
4902 case RISCV::BGEU:
4903 // bgeu zero, rs, imm => beq rs, zero, imm
4904 if (MI.getOperand(i: 0).getReg() == RISCV::X0) {
4905 MachineOperand MO0 = MI.getOperand(i: 0);
4906 MI.removeOperand(OpNo: 0);
4907 MI.insert(InsertBefore: MI.operands_begin() + 1, Ops: {MO0});
4908 MI.setDesc(get(Opcode: RISCV::BEQ));
4909 }
4910 break;
4911 }
4912 return false;
4913}
4914
4915// clang-format off
4916#define CASE_WIDEOP_OPCODE_COMMON(OP, LMUL) \
4917 RISCV::PseudoV##OP##_##LMUL##_TIED
4918
4919#define CASE_WIDEOP_OPCODE_LMULS(OP) \
4920 CASE_WIDEOP_OPCODE_COMMON(OP, MF8): \
4921 case CASE_WIDEOP_OPCODE_COMMON(OP, MF4): \
4922 case CASE_WIDEOP_OPCODE_COMMON(OP, MF2): \
4923 case CASE_WIDEOP_OPCODE_COMMON(OP, M1): \
4924 case CASE_WIDEOP_OPCODE_COMMON(OP, M2): \
4925 case CASE_WIDEOP_OPCODE_COMMON(OP, M4)
4926
4927#define CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL) \
4928 case RISCV::PseudoV##OP##_##LMUL##_TIED: \
4929 NewOpc = RISCV::PseudoV##OP##_##LMUL; \
4930 break;
4931
4932#define CASE_WIDEOP_CHANGE_OPCODE_LMULS(OP) \
4933 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF8) \
4934 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4) \
4935 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2) \
4936 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1) \
4937 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2) \
4938 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4)
4939
4940// FP Widening Ops may by SEW aware. Create SEW aware cases for these cases.
4941#define CASE_FP_WIDEOP_OPCODE_COMMON(OP, LMUL, SEW) \
4942 RISCV::PseudoV##OP##_##LMUL##_##SEW##_TIED
4943
4944#define CASE_FP_WIDEOP_OPCODE_LMULS(OP) \
4945 CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF4, E16): \
4946 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E16): \
4947 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E32): \
4948 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E16): \
4949 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E32): \
4950 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E16): \
4951 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E32): \
4952 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E16): \
4953 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E32) \
4954
4955#define CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL, SEW) \
4956 case RISCV::PseudoV##OP##_##LMUL##_##SEW##_TIED: \
4957 NewOpc = RISCV::PseudoV##OP##_##LMUL##_##SEW; \
4958 break;
4959
4960#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(OP) \
4961 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4, E16) \
4962 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E16) \
4963 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E32) \
4964 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E16) \
4965 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E32) \
4966 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E16) \
4967 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E32) \
4968 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E16) \
4969 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E32) \
4970
4971#define CASE_FP_WIDEOP_OPCODE_LMULS_ALT(OP) \
4972 CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF4, E16): \
4973 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E16): \
4974 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E16): \
4975 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E16): \
4976 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E16)
4977
4978#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_ALT(OP) \
4979 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4, E16) \
4980 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E16) \
4981 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E16) \
4982 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E16) \
4983 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E16)
4984// clang-format on
4985
4986MachineInstr *RISCVInstrInfo::convertToThreeAddress(MachineInstr &MI,
4987 LiveIntervals *LIS) const {
4988 MachineInstrBuilder MIB;
4989 switch (MI.getOpcode()) {
4990 default:
4991 return nullptr;
4992 case CASE_FP_WIDEOP_OPCODE_LMULS_ALT(FWADD_ALT_WV):
4993 case CASE_FP_WIDEOP_OPCODE_LMULS_ALT(FWSUB_ALT_WV):
4994 case CASE_FP_WIDEOP_OPCODE_LMULS(FWADD_WV):
4995 case CASE_FP_WIDEOP_OPCODE_LMULS(FWSUB_WV): {
4996 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags) &&
4997 MI.getNumExplicitOperands() == 7 &&
4998 "Expect 7 explicit operands rd, rs2, rs1, rm, vl, sew, policy");
4999 // If the tail policy is undisturbed we can't convert.
5000 if ((MI.getOperand(i: RISCVII::getVecPolicyOpNum(Desc: MI.getDesc())).getImm() &
5001 1) == 0)
5002 return nullptr;
5003 // clang-format off
5004 unsigned NewOpc;
5005 switch (MI.getOpcode()) {
5006 default:
5007 llvm_unreachable("Unexpected opcode");
5008 CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(FWADD_WV)
5009 CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(FWSUB_WV)
5010 CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_ALT(FWADD_ALT_WV)
5011 CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_ALT(FWSUB_ALT_WV)
5012 }
5013 // clang-format on
5014
5015 MachineBasicBlock &MBB = *MI.getParent();
5016 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
5017 .add(MO: MI.getOperand(i: 0))
5018 .addReg(RegNo: MI.getOperand(i: 0).getReg(), Flags: RegState::Undef)
5019 .add(MO: MI.getOperand(i: 1))
5020 .add(MO: MI.getOperand(i: 2))
5021 .add(MO: MI.getOperand(i: 3))
5022 .add(MO: MI.getOperand(i: 4))
5023 .add(MO: MI.getOperand(i: 5))
5024 .add(MO: MI.getOperand(i: 6));
5025 break;
5026 }
5027 case CASE_WIDEOP_OPCODE_LMULS(WADD_WV):
5028 case CASE_WIDEOP_OPCODE_LMULS(WADDU_WV):
5029 case CASE_WIDEOP_OPCODE_LMULS(WSUB_WV):
5030 case CASE_WIDEOP_OPCODE_LMULS(WSUBU_WV): {
5031 // If the tail policy is undisturbed we can't convert.
5032 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags) &&
5033 MI.getNumExplicitOperands() == 6);
5034 if ((MI.getOperand(i: RISCVII::getVecPolicyOpNum(Desc: MI.getDesc())).getImm() &
5035 1) == 0)
5036 return nullptr;
5037
5038 // clang-format off
5039 unsigned NewOpc;
5040 switch (MI.getOpcode()) {
5041 default:
5042 llvm_unreachable("Unexpected opcode");
5043 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WADD_WV)
5044 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WADDU_WV)
5045 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WSUB_WV)
5046 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WSUBU_WV)
5047 }
5048 // clang-format on
5049
5050 MachineBasicBlock &MBB = *MI.getParent();
5051 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
5052 .add(MO: MI.getOperand(i: 0))
5053 .addReg(RegNo: MI.getOperand(i: 0).getReg(), Flags: RegState::Undef)
5054 .add(MO: MI.getOperand(i: 1))
5055 .add(MO: MI.getOperand(i: 2))
5056 .add(MO: MI.getOperand(i: 3))
5057 .add(MO: MI.getOperand(i: 4))
5058 .add(MO: MI.getOperand(i: 5));
5059 break;
5060 }
5061 }
5062 MIB.copyImplicitOps(OtherMI: MI);
5063
5064 if (LIS) {
5065 SlotIndex Idx = LIS->ReplaceMachineInstrInMaps(MI, NewMI&: *MIB);
5066
5067 if (MI.getOperand(i: 0).isEarlyClobber()) {
5068 // Use operand 1 was tied to early-clobber def operand 0, so its live
5069 // interval could have ended at an early-clobber slot. Now they are not
5070 // tied we need to update it to the normal register slot.
5071 LiveInterval &LI = LIS->getInterval(Reg: MI.getOperand(i: 1).getReg());
5072 LiveRange::Segment *S = LI.getSegmentContaining(Idx);
5073 if (S->end == Idx.getRegSlot(EC: true))
5074 S->end = Idx.getRegSlot();
5075 }
5076 }
5077
5078 return MIB;
5079}
5080
5081#undef CASE_WIDEOP_OPCODE_COMMON
5082#undef CASE_WIDEOP_OPCODE_LMULS
5083#undef CASE_WIDEOP_CHANGE_OPCODE_COMMON
5084#undef CASE_WIDEOP_CHANGE_OPCODE_LMULS
5085#undef CASE_FP_WIDEOP_OPCODE_COMMON
5086#undef CASE_FP_WIDEOP_OPCODE_LMULS
5087#undef CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON
5088#undef CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS
5089
5090void RISCVInstrInfo::mulImm(MachineFunction &MF, MachineBasicBlock &MBB,
5091 MachineBasicBlock::iterator II, const DebugLoc &DL,
5092 Register DestReg, uint32_t Amount,
5093 MachineInstr::MIFlag Flag) const {
5094 MachineRegisterInfo &MRI = MF.getRegInfo();
5095 if (llvm::has_single_bit(Value: Amount)) {
5096 uint32_t ShiftAmount = Log2_32(Value: Amount);
5097 if (ShiftAmount == 0)
5098 return;
5099 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::SLLI), DestReg)
5100 .addReg(RegNo: DestReg, Flags: RegState::Kill)
5101 .addImm(Val: ShiftAmount)
5102 .setMIFlag(Flag);
5103 } else if (int ShXAmount, ShiftAmount;
5104 STI.hasStdExtZba() &&
5105 (ShXAmount = isShifted359(Value: Amount, Shift&: ShiftAmount)) != 0) {
5106 // We can use Zba SHXADD+SLLI instructions for multiply in some cases.
5107 unsigned Opc;
5108 switch (ShXAmount) {
5109 case 1:
5110 Opc = RISCV::SH1ADD;
5111 break;
5112 case 2:
5113 Opc = RISCV::SH2ADD;
5114 break;
5115 case 3:
5116 Opc = RISCV::SH3ADD;
5117 break;
5118 default:
5119 llvm_unreachable("unexpected result of isShifted359");
5120 }
5121 if (ShiftAmount)
5122 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::SLLI), DestReg)
5123 .addReg(RegNo: DestReg, Flags: RegState::Kill)
5124 .addImm(Val: ShiftAmount)
5125 .setMIFlag(Flag);
5126 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: Opc), DestReg)
5127 .addReg(RegNo: DestReg, Flags: RegState::Kill)
5128 .addReg(RegNo: DestReg)
5129 .setMIFlag(Flag);
5130 } else if (llvm::has_single_bit(Value: Amount - 1)) {
5131 Register ScaledRegister = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
5132 uint32_t ShiftAmount = Log2_32(Value: Amount - 1);
5133 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::SLLI), DestReg: ScaledRegister)
5134 .addReg(RegNo: DestReg)
5135 .addImm(Val: ShiftAmount)
5136 .setMIFlag(Flag);
5137 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::ADD), DestReg)
5138 .addReg(RegNo: ScaledRegister, Flags: RegState::Kill)
5139 .addReg(RegNo: DestReg, Flags: RegState::Kill)
5140 .setMIFlag(Flag);
5141 } else if (llvm::has_single_bit(Value: Amount + 1)) {
5142 Register ScaledRegister = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
5143 uint32_t ShiftAmount = Log2_32(Value: Amount + 1);
5144 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::SLLI), DestReg: ScaledRegister)
5145 .addReg(RegNo: DestReg)
5146 .addImm(Val: ShiftAmount)
5147 .setMIFlag(Flag);
5148 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::SUB), DestReg)
5149 .addReg(RegNo: ScaledRegister, Flags: RegState::Kill)
5150 .addReg(RegNo: DestReg, Flags: RegState::Kill)
5151 .setMIFlag(Flag);
5152 } else if (STI.hasStdExtZmmul()) {
5153 Register N = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
5154 movImm(MBB, MBBI: II, DL, DstReg: N, Val: Amount, Flag);
5155 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::MUL), DestReg)
5156 .addReg(RegNo: DestReg, Flags: RegState::Kill)
5157 .addReg(RegNo: N, Flags: RegState::Kill)
5158 .setMIFlag(Flag);
5159 } else {
5160 Register Acc;
5161 uint32_t PrevShiftAmount = 0;
5162 for (uint32_t ShiftAmount = 0; Amount >> ShiftAmount; ShiftAmount++) {
5163 if (Amount & (1U << ShiftAmount)) {
5164 if (ShiftAmount)
5165 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::SLLI), DestReg)
5166 .addReg(RegNo: DestReg, Flags: RegState::Kill)
5167 .addImm(Val: ShiftAmount - PrevShiftAmount)
5168 .setMIFlag(Flag);
5169 if (Amount >> (ShiftAmount + 1)) {
5170 // If we don't have an accmulator yet, create it and copy DestReg.
5171 if (!Acc) {
5172 Acc = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
5173 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: TargetOpcode::COPY), DestReg: Acc)
5174 .addReg(RegNo: DestReg)
5175 .setMIFlag(Flag);
5176 } else {
5177 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::ADD), DestReg: Acc)
5178 .addReg(RegNo: Acc, Flags: RegState::Kill)
5179 .addReg(RegNo: DestReg)
5180 .setMIFlag(Flag);
5181 }
5182 }
5183 PrevShiftAmount = ShiftAmount;
5184 }
5185 }
5186 assert(Acc && "Expected valid accumulator");
5187 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::ADD), DestReg)
5188 .addReg(RegNo: DestReg, Flags: RegState::Kill)
5189 .addReg(RegNo: Acc, Flags: RegState::Kill)
5190 .setMIFlag(Flag);
5191 }
5192}
5193
5194ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
5195RISCVInstrInfo::getSerializableMachineMemOperandTargetFlags() const {
5196 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
5197 {{MONontemporalBit0, "riscv-nontemporal-domain-bit-0"},
5198 {MONontemporalBit1, "riscv-nontemporal-domain-bit-1"}};
5199 return ArrayRef(TargetFlags);
5200}
5201
5202unsigned RISCVInstrInfo::getTailDuplicateSize(CodeGenOptLevel OptLevel) const {
5203 return OptLevel >= CodeGenOptLevel::Aggressive
5204 ? STI.getTailDupAggressiveThreshold()
5205 : 2;
5206}
5207
5208bool RISCV::isRVVSpill(const MachineInstr &MI) {
5209 // RVV lacks any support for immediate addressing for stack addresses, so be
5210 // conservative.
5211 unsigned Opcode = MI.getOpcode();
5212 if (!RISCVVPseudosTable::getPseudoInfo(Pseudo: Opcode) &&
5213 !getLMULForRVVWholeLoadStore(Opcode) && !isRVVSpillForZvlsseg(Opcode))
5214 return false;
5215 return true;
5216}
5217
5218/// Return true if \p MI is a copy that will be lowered to one or more vmvNr.vs.
5219bool RISCV::isVectorCopy(const TargetRegisterInfo *TRI,
5220 const MachineInstr &MI) {
5221 return MI.isCopy() && MI.getOperand(i: 0).getReg().isPhysical() &&
5222 RISCVRegisterInfo::isRVVRegClass(
5223 RC: TRI->getMinimalPhysRegClass(Reg: MI.getOperand(i: 0).getReg()));
5224}
5225
5226std::optional<std::pair<unsigned, unsigned>>
5227RISCV::isRVVSpillForZvlsseg(unsigned Opcode) {
5228 switch (Opcode) {
5229 default:
5230 return std::nullopt;
5231 case RISCV::PseudoVSPILL2_M1:
5232 case RISCV::PseudoVRELOAD2_M1:
5233 return std::make_pair(x: 2u, y: 1u);
5234 case RISCV::PseudoVSPILL2_M2:
5235 case RISCV::PseudoVRELOAD2_M2:
5236 return std::make_pair(x: 2u, y: 2u);
5237 case RISCV::PseudoVSPILL2_M4:
5238 case RISCV::PseudoVRELOAD2_M4:
5239 return std::make_pair(x: 2u, y: 4u);
5240 case RISCV::PseudoVSPILL3_M1:
5241 case RISCV::PseudoVRELOAD3_M1:
5242 return std::make_pair(x: 3u, y: 1u);
5243 case RISCV::PseudoVSPILL3_M2:
5244 case RISCV::PseudoVRELOAD3_M2:
5245 return std::make_pair(x: 3u, y: 2u);
5246 case RISCV::PseudoVSPILL4_M1:
5247 case RISCV::PseudoVRELOAD4_M1:
5248 return std::make_pair(x: 4u, y: 1u);
5249 case RISCV::PseudoVSPILL4_M2:
5250 case RISCV::PseudoVRELOAD4_M2:
5251 return std::make_pair(x: 4u, y: 2u);
5252 case RISCV::PseudoVSPILL5_M1:
5253 case RISCV::PseudoVRELOAD5_M1:
5254 return std::make_pair(x: 5u, y: 1u);
5255 case RISCV::PseudoVSPILL6_M1:
5256 case RISCV::PseudoVRELOAD6_M1:
5257 return std::make_pair(x: 6u, y: 1u);
5258 case RISCV::PseudoVSPILL7_M1:
5259 case RISCV::PseudoVRELOAD7_M1:
5260 return std::make_pair(x: 7u, y: 1u);
5261 case RISCV::PseudoVSPILL8_M1:
5262 case RISCV::PseudoVRELOAD8_M1:
5263 return std::make_pair(x: 8u, y: 1u);
5264 }
5265}
5266
5267bool RISCV::hasEqualFRM(const MachineInstr &MI1, const MachineInstr &MI2) {
5268 int16_t MI1FrmOpIdx =
5269 RISCV::getNamedOperandIdx(Opcode: MI1.getOpcode(), Name: RISCV::OpName::frm);
5270 int16_t MI2FrmOpIdx =
5271 RISCV::getNamedOperandIdx(Opcode: MI2.getOpcode(), Name: RISCV::OpName::frm);
5272 if (MI1FrmOpIdx < 0 || MI2FrmOpIdx < 0)
5273 return false;
5274 MachineOperand FrmOp1 = MI1.getOperand(i: MI1FrmOpIdx);
5275 MachineOperand FrmOp2 = MI2.getOperand(i: MI2FrmOpIdx);
5276 return FrmOp1.getImm() == FrmOp2.getImm();
5277}
5278
5279std::optional<unsigned>
5280RISCV::getVectorLowDemandedScalarBits(unsigned Opcode, unsigned Log2SEW) {
5281 switch (Opcode) {
5282 default:
5283 return std::nullopt;
5284
5285 // 11.6. Vector Single-Width Shift Instructions
5286 case RISCV::VSLL_VX:
5287 case RISCV::VSRL_VX:
5288 case RISCV::VSRA_VX:
5289 // 12.4. Vector Single-Width Scaling Shift Instructions
5290 case RISCV::VSSRL_VX:
5291 case RISCV::VSSRA_VX:
5292 // Zvbb
5293 case RISCV::VROL_VX:
5294 case RISCV::VROR_VX:
5295 // Only the low lg2(SEW) bits of the shift-amount value are used.
5296 return Log2SEW;
5297
5298 // 11.7 Vector Narrowing Integer Right Shift Instructions
5299 case RISCV::VNSRL_WX:
5300 case RISCV::VNSRA_WX:
5301 // 12.5. Vector Narrowing Fixed-Point Clip Instructions
5302 case RISCV::VNCLIPU_WX:
5303 case RISCV::VNCLIP_WX:
5304 // Zvbb
5305 case RISCV::VWSLL_VX:
5306 // Only the low lg2(2*SEW) bits of the shift-amount value are used.
5307 return Log2SEW + 1;
5308
5309 // 11.1. Vector Single-Width Integer Add and Subtract
5310 case RISCV::VADD_VX:
5311 case RISCV::VSUB_VX:
5312 case RISCV::VRSUB_VX:
5313 // 11.2. Vector Widening Integer Add/Subtract
5314 case RISCV::VWADDU_VX:
5315 case RISCV::VWSUBU_VX:
5316 case RISCV::VWADD_VX:
5317 case RISCV::VWSUB_VX:
5318 case RISCV::VWADDU_WX:
5319 case RISCV::VWSUBU_WX:
5320 case RISCV::VWADD_WX:
5321 case RISCV::VWSUB_WX:
5322 // 11.4. Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions
5323 case RISCV::VADC_VXM:
5324 case RISCV::VADC_VIM:
5325 case RISCV::VMADC_VXM:
5326 case RISCV::VMADC_VIM:
5327 case RISCV::VMADC_VX:
5328 case RISCV::VSBC_VXM:
5329 case RISCV::VMSBC_VXM:
5330 case RISCV::VMSBC_VX:
5331 // 11.5 Vector Bitwise Logical Instructions
5332 case RISCV::VAND_VX:
5333 case RISCV::VOR_VX:
5334 case RISCV::VXOR_VX:
5335 // 11.8. Vector Integer Compare Instructions
5336 case RISCV::VMSEQ_VX:
5337 case RISCV::VMSNE_VX:
5338 case RISCV::VMSLTU_VX:
5339 case RISCV::VMSLT_VX:
5340 case RISCV::VMSLEU_VX:
5341 case RISCV::VMSLE_VX:
5342 case RISCV::VMSGTU_VX:
5343 case RISCV::VMSGT_VX:
5344 // 11.9. Vector Integer Min/Max Instructions
5345 case RISCV::VMINU_VX:
5346 case RISCV::VMIN_VX:
5347 case RISCV::VMAXU_VX:
5348 case RISCV::VMAX_VX:
5349 // 11.10. Vector Single-Width Integer Multiply Instructions
5350 case RISCV::VMUL_VX:
5351 case RISCV::VMULH_VX:
5352 case RISCV::VMULHU_VX:
5353 case RISCV::VMULHSU_VX:
5354 // 11.11. Vector Integer Divide Instructions
5355 case RISCV::VDIVU_VX:
5356 case RISCV::VDIV_VX:
5357 case RISCV::VREMU_VX:
5358 case RISCV::VREM_VX:
5359 // 11.12. Vector Widening Integer Multiply Instructions
5360 case RISCV::VWMUL_VX:
5361 case RISCV::VWMULU_VX:
5362 case RISCV::VWMULSU_VX:
5363 // 11.13. Vector Single-Width Integer Multiply-Add Instructions
5364 case RISCV::VMACC_VX:
5365 case RISCV::VNMSAC_VX:
5366 case RISCV::VMADD_VX:
5367 case RISCV::VNMSUB_VX:
5368 // 11.14. Vector Widening Integer Multiply-Add Instructions
5369 case RISCV::VWMACCU_VX:
5370 case RISCV::VWMACC_VX:
5371 case RISCV::VWMACCSU_VX:
5372 case RISCV::VWMACCUS_VX:
5373 // 11.15. Vector Integer Merge Instructions
5374 case RISCV::VMERGE_VXM:
5375 // 11.16. Vector Integer Move Instructions
5376 case RISCV::VMV_V_X:
5377 // 12.1. Vector Single-Width Saturating Add and Subtract
5378 case RISCV::VSADDU_VX:
5379 case RISCV::VSADD_VX:
5380 case RISCV::VSSUBU_VX:
5381 case RISCV::VSSUB_VX:
5382 // 12.2. Vector Single-Width Averaging Add and Subtract
5383 case RISCV::VAADDU_VX:
5384 case RISCV::VAADD_VX:
5385 case RISCV::VASUBU_VX:
5386 case RISCV::VASUB_VX:
5387 // 12.3. Vector Single-Width Fractional Multiply with Rounding and Saturation
5388 case RISCV::VSMUL_VX:
5389 // 16.1. Integer Scalar Move Instructions
5390 case RISCV::VMV_S_X:
5391 // Zvbb
5392 case RISCV::VANDN_VX:
5393 return 1U << Log2SEW;
5394 }
5395}
5396
5397unsigned RISCV::getRVVMCOpcode(unsigned RVVPseudoOpcode) {
5398 const RISCVVPseudosTable::PseudoInfo *RVV =
5399 RISCVVPseudosTable::getPseudoInfo(Pseudo: RVVPseudoOpcode);
5400 if (!RVV)
5401 return 0;
5402 return RVV->BaseInstr;
5403}
5404
5405unsigned RISCV::getDestLog2EEW(const MCInstrDesc &Desc, unsigned Log2SEW) {
5406 unsigned DestEEW =
5407 (Desc.TSFlags & RISCVII::DestEEWMask) >> RISCVII::DestEEWShift;
5408 // EEW = 1
5409 if (DestEEW == 0)
5410 return 0;
5411 // EEW = SEW * n
5412 unsigned Scaled = Log2SEW + (DestEEW - 1);
5413 assert(Scaled >= 3 && Scaled <= 6);
5414 return Scaled;
5415}
5416
5417static std::optional<int64_t> getEffectiveImm(const MachineRegisterInfo &MRI,
5418 const MachineOperand &MO) {
5419 assert(MO.isImm() || MO.getReg().isVirtual());
5420 if (MO.isImm())
5421 return MO.getImm();
5422 const MachineInstr *Def = MRI.getVRegDef(Reg: MO.getReg());
5423 int64_t Imm;
5424 if (isLoadImm(MI: Def, Imm))
5425 return Imm;
5426 return std::nullopt;
5427}
5428
5429/// Given two VL operands, do we know that LHS <= RHS? Must be used in SSA form.
5430bool RISCV::isVLKnownLE(const MachineRegisterInfo &MRI,
5431 const MachineOperand &LHS, const MachineOperand &RHS) {
5432 assert((LHS.isImm() || MRI.isSSA()) && (RHS.isImm() || MRI.isSSA()));
5433 if (LHS.isReg() && RHS.isReg() && LHS.getReg().isVirtual() &&
5434 LHS.getReg() == RHS.getReg())
5435 return true;
5436 if (RHS.isImm() && RHS.getImm() == RISCV::VLMaxSentinel)
5437 return true;
5438 if (LHS.isImm() && LHS.getImm() == 0)
5439 return true;
5440 if (LHS.isImm() && LHS.getImm() == RISCV::VLMaxSentinel)
5441 return false;
5442 std::optional<int64_t> LHSImm = getEffectiveImm(MRI, MO: LHS),
5443 RHSImm = getEffectiveImm(MRI, MO: RHS);
5444 if (!LHSImm || !RHSImm)
5445 return false;
5446 return LHSImm <= RHSImm;
5447}
5448
5449namespace {
5450class RISCVPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
5451 const MachineInstr *LHS;
5452 const MachineInstr *RHS;
5453 SmallVector<MachineOperand, 3> Cond;
5454
5455public:
5456 RISCVPipelinerLoopInfo(const MachineInstr *LHS, const MachineInstr *RHS,
5457 const SmallVectorImpl<MachineOperand> &Cond)
5458 : LHS(LHS), RHS(RHS), Cond(Cond.begin(), Cond.end()) {}
5459
5460 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
5461 // Make the instructions for loop control be placed in stage 0.
5462 // The predecessors of LHS/RHS are considered by the caller.
5463 if (LHS && MI == LHS)
5464 return true;
5465 if (RHS && MI == RHS)
5466 return true;
5467 return false;
5468 }
5469
5470 std::optional<bool> createTripCountGreaterCondition(
5471 int TC, MachineBasicBlock &MBB,
5472 SmallVectorImpl<MachineOperand> &CondParam) override {
5473 // A branch instruction will be inserted as "if (Cond) goto epilogue".
5474 // Cond is normalized for such use.
5475 // The predecessors of the branch are assumed to have already been inserted.
5476 CondParam = Cond;
5477 return {};
5478 }
5479
5480 void setPreheader(MachineBasicBlock *NewPreheader) override {}
5481
5482 void adjustTripCount(int TripCountAdjust) override {}
5483};
5484} // namespace
5485
5486std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
5487RISCVInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
5488 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
5489 SmallVector<MachineOperand, 4> Cond;
5490 if (analyzeBranch(MBB&: *LoopBB, TBB, FBB, Cond, /*AllowModify=*/false))
5491 return nullptr;
5492
5493 // Infinite loops are not supported
5494 if (TBB == LoopBB && FBB == LoopBB)
5495 return nullptr;
5496
5497 // Must be conditional branch
5498 if (FBB == nullptr)
5499 return nullptr;
5500
5501 assert((TBB == LoopBB || FBB == LoopBB) &&
5502 "The Loop must be a single-basic-block loop");
5503
5504 // Normalization for createTripCountGreaterCondition()
5505 if (TBB == LoopBB)
5506 reverseBranchCondition(Cond);
5507
5508 const MachineRegisterInfo &MRI = LoopBB->getParent()->getRegInfo();
5509 auto FindRegDef = [&MRI](MachineOperand &Op) -> const MachineInstr * {
5510 if (!Op.isReg())
5511 return nullptr;
5512 Register Reg = Op.getReg();
5513 if (!Reg.isVirtual())
5514 return nullptr;
5515 return MRI.getVRegDef(Reg);
5516 };
5517
5518 const MachineInstr *LHS = FindRegDef(Cond[1]);
5519 const MachineInstr *RHS = FindRegDef(Cond[2]);
5520 if (LHS && LHS->isPHI())
5521 return nullptr;
5522 if (RHS && RHS->isPHI())
5523 return nullptr;
5524
5525 return std::make_unique<RISCVPipelinerLoopInfo>(args&: LHS, args&: RHS, args&: Cond);
5526}
5527
5528// FIXME: We should remove this if we have a default generic scheduling model.
5529bool RISCVInstrInfo::isHighLatencyDef(int Opc) const {
5530 unsigned RVVMCOpcode = RISCV::getRVVMCOpcode(RVVPseudoOpcode: Opc);
5531 Opc = RVVMCOpcode ? RVVMCOpcode : Opc;
5532 switch (Opc) {
5533 default:
5534 return false;
5535 // Integer div/rem.
5536 case RISCV::DIV:
5537 case RISCV::DIVW:
5538 case RISCV::DIVU:
5539 case RISCV::DIVUW:
5540 case RISCV::REM:
5541 case RISCV::REMW:
5542 case RISCV::REMU:
5543 case RISCV::REMUW:
5544 // Floating-point div/sqrt.
5545 case RISCV::FDIV_H:
5546 case RISCV::FDIV_S:
5547 case RISCV::FDIV_D:
5548 case RISCV::FDIV_H_INX:
5549 case RISCV::FDIV_S_INX:
5550 case RISCV::FDIV_D_INX:
5551 case RISCV::FDIV_D_IN32X:
5552 case RISCV::FSQRT_H:
5553 case RISCV::FSQRT_S:
5554 case RISCV::FSQRT_D:
5555 case RISCV::FSQRT_H_INX:
5556 case RISCV::FSQRT_S_INX:
5557 case RISCV::FSQRT_D_INX:
5558 case RISCV::FSQRT_D_IN32X:
5559 // Vector integer div/rem
5560 case RISCV::VDIV_VV:
5561 case RISCV::VDIV_VX:
5562 case RISCV::VDIVU_VV:
5563 case RISCV::VDIVU_VX:
5564 case RISCV::VREM_VV:
5565 case RISCV::VREM_VX:
5566 case RISCV::VREMU_VV:
5567 case RISCV::VREMU_VX:
5568 // Vector floating-point div/sqrt.
5569 case RISCV::VFDIV_VV:
5570 case RISCV::VFDIV_VF:
5571 case RISCV::VFRDIV_VF:
5572 case RISCV::VFSQRT_V:
5573 case RISCV::VFRSQRT7_V:
5574 return true;
5575 }
5576}
5577
5578bool RISCVInstrInfo::isVRegCopy(const MachineInstr *MI, unsigned LMul) const {
5579 if (MI->getOpcode() != TargetOpcode::COPY)
5580 return false;
5581 const MachineRegisterInfo &MRI = MI->getMF()->getRegInfo();
5582 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
5583
5584 Register DstReg = MI->getOperand(i: 0).getReg();
5585 const TargetRegisterClass *RC = DstReg.isVirtual()
5586 ? MRI.getRegClass(Reg: DstReg)
5587 : TRI->getMinimalPhysRegClass(Reg: DstReg);
5588
5589 if (!RISCVRegisterInfo::isRVVRegClass(RC))
5590 return false;
5591
5592 if (!LMul)
5593 return true;
5594
5595 // TODO: Perhaps we could distinguish segment register classes (e.g. VRN3M2)
5596 // in the future.
5597 auto [RCLMul, RCFractional] =
5598 RISCVVType::decodeVLMUL(VLMul: RISCVRI::getLMul(TSFlags: RC->TSFlags));
5599 return (!RCFractional && LMul == RCLMul) || (RCFractional && LMul == 1);
5600}
5601
5602bool RISCVInstrInfo::requiresNTLHint(const MachineInstr &MI) const {
5603 if (MI.memoperands_empty())
5604 return false;
5605
5606 MachineMemOperand *MMO = *(MI.memoperands_begin());
5607 if (!MMO->isNonTemporal())
5608 return false;
5609
5610 return true;
5611}
5612
5613bool RISCVInstrInfo::isSafeToMove(const MachineInstr &From,
5614 const MachineBasicBlock::iterator &To) {
5615 assert(To == From.getParent()->end() || From.getParent() == To->getParent());
5616 SmallVector<Register> PhysUses, PhysDefs;
5617 for (const MachineOperand &MO : From.all_uses())
5618 if (MO.getReg().isPhysical())
5619 PhysUses.push_back(Elt: MO.getReg());
5620 for (const MachineOperand &MO : From.all_defs())
5621 if (MO.getReg().isPhysical())
5622 PhysDefs.push_back(Elt: MO.getReg());
5623 bool SawStore = false;
5624 for (auto II = std::next(x: From.getIterator()); II != To; II++) {
5625 for (Register PhysReg : PhysUses)
5626 if (II->definesRegister(Reg: PhysReg, TRI: nullptr))
5627 return false;
5628 for (Register PhysReg : PhysDefs)
5629 if (II->definesRegister(Reg: PhysReg, TRI: nullptr) ||
5630 II->readsRegister(Reg: PhysReg, TRI: nullptr))
5631 return false;
5632 II->isSafeToMove(SawStore);
5633 if (SawStore)
5634 break;
5635 }
5636 return From.isSafeToMove(SawStore);
5637}
5638