1//===-- RISCVInstrInfo.cpp - RISC-V Instruction Information -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the RISC-V implementation of the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RISCVInstrInfo.h"
14#include "MCTargetDesc/RISCVBaseInfo.h"
15#include "MCTargetDesc/RISCVMatInt.h"
16#include "RISCV.h"
17#include "RISCVMachineFunctionInfo.h"
18#include "RISCVSubtarget.h"
19#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/ADT/Statistic.h"
22#include "llvm/Analysis/MemoryLocation.h"
23#include "llvm/Analysis/ValueTracking.h"
24#include "llvm/CodeGen/LiveIntervals.h"
25#include "llvm/CodeGen/LiveVariables.h"
26#include "llvm/CodeGen/MachineCombinerPattern.h"
27#include "llvm/CodeGen/MachineInstrBuilder.h"
28#include "llvm/CodeGen/MachineRegisterInfo.h"
29#include "llvm/CodeGen/MachineTraceMetrics.h"
30#include "llvm/CodeGen/RegisterScavenging.h"
31#include "llvm/CodeGen/StackMaps.h"
32#include "llvm/IR/DebugInfoMetadata.h"
33#include "llvm/IR/Module.h"
34#include "llvm/MC/MCDwarf.h"
35#include "llvm/MC/MCInstBuilder.h"
36#include "llvm/MC/TargetRegistry.h"
37#include "llvm/Support/ErrorHandling.h"
38
39using namespace llvm;
40
41#define GEN_CHECK_COMPRESS_INSTR
42#include "RISCVGenCompressInstEmitter.inc"
43
44#define GET_INSTRINFO_CTOR_DTOR
45#define GET_INSTRINFO_NAMED_OPS
46#include "RISCVGenInstrInfo.inc"
47
48#define DEBUG_TYPE "riscv-instr-info"
49STATISTIC(NumVRegSpilled,
50 "Number of registers within vector register groups spilled");
51STATISTIC(NumVRegReloaded,
52 "Number of registers within vector register groups reloaded");
53
54static cl::opt<bool> PreferWholeRegisterMove(
55 "riscv-prefer-whole-register-move", cl::init(Val: false), cl::Hidden,
56 cl::desc("Prefer whole register move for vector registers."));
57
58static cl::opt<MachineTraceStrategy> ForceMachineCombinerStrategy(
59 "riscv-force-machine-combiner-strategy", cl::Hidden,
60 cl::desc("Force machine combiner to use a specific strategy for machine "
61 "trace metrics evaluation."),
62 cl::init(Val: MachineTraceStrategy::TS_NumStrategies),
63 cl::values(clEnumValN(MachineTraceStrategy::TS_Local, "local",
64 "Local strategy."),
65 clEnumValN(MachineTraceStrategy::TS_MinInstrCount, "min-instr",
66 "MinInstrCount strategy.")));
67
68namespace llvm::RISCVVPseudosTable {
69
70using namespace RISCV;
71
72#define GET_RISCVVPseudosTable_IMPL
73#include "RISCVGenSearchableTables.inc"
74
75} // namespace llvm::RISCVVPseudosTable
76
77namespace llvm::RISCV {
78
79#define GET_RISCVMaskedPseudosTable_IMPL
80#include "RISCVGenSearchableTables.inc"
81
82} // end namespace llvm::RISCV
83
84RISCVInstrInfo::RISCVInstrInfo(const RISCVSubtarget &STI)
85 : RISCVGenInstrInfo(STI, RegInfo, RISCV::ADJCALLSTACKDOWN,
86 RISCV::ADJCALLSTACKUP),
87 RegInfo(STI.getHwMode()), STI(STI) {}
88
89#define GET_INSTRINFO_HELPERS
90#include "RISCVGenInstrInfo.inc"
91
92MCInst RISCVInstrInfo::getNop() const {
93 if (STI.hasStdExtZca())
94 return MCInstBuilder(RISCV::C_NOP);
95 return MCInstBuilder(RISCV::ADDI)
96 .addReg(Reg: RISCV::X0)
97 .addReg(Reg: RISCV::X0)
98 .addImm(Val: 0);
99}
100
101Register RISCVInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
102 int &FrameIndex) const {
103 TypeSize Dummy = TypeSize::getZero();
104 return isLoadFromStackSlot(MI, FrameIndex, MemBytes&: Dummy);
105}
106
107static std::optional<unsigned> getLMULForRVVWholeLoadStore(unsigned Opcode) {
108 switch (Opcode) {
109 default:
110 return std::nullopt;
111 case RISCV::VS1R_V:
112 case RISCV::VL1RE8_V:
113 case RISCV::VL1RE16_V:
114 case RISCV::VL1RE32_V:
115 case RISCV::VL1RE64_V:
116 return 1;
117 case RISCV::VS2R_V:
118 case RISCV::VL2RE8_V:
119 case RISCV::VL2RE16_V:
120 case RISCV::VL2RE32_V:
121 case RISCV::VL2RE64_V:
122 return 2;
123 case RISCV::VS4R_V:
124 case RISCV::VL4RE8_V:
125 case RISCV::VL4RE16_V:
126 case RISCV::VL4RE32_V:
127 case RISCV::VL4RE64_V:
128 return 4;
129 case RISCV::VS8R_V:
130 case RISCV::VL8RE8_V:
131 case RISCV::VL8RE16_V:
132 case RISCV::VL8RE32_V:
133 case RISCV::VL8RE64_V:
134 return 8;
135 }
136}
137
138Register RISCVInstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
139 int &FrameIndex,
140 TypeSize &MemBytes) const {
141 switch (MI.getOpcode()) {
142 default:
143 return 0;
144 case RISCV::LB:
145 case RISCV::LBU:
146 MemBytes = TypeSize::getFixed(ExactSize: 1);
147 break;
148 case RISCV::LH:
149 case RISCV::LH_INX:
150 case RISCV::LHU:
151 case RISCV::FLH:
152 MemBytes = TypeSize::getFixed(ExactSize: 2);
153 break;
154 case RISCV::LW:
155 case RISCV::LW_INX:
156 case RISCV::FLW:
157 case RISCV::LWU:
158 MemBytes = TypeSize::getFixed(ExactSize: 4);
159 break;
160 case RISCV::LD:
161 case RISCV::LD_RV32:
162 case RISCV::FLD:
163 MemBytes = TypeSize::getFixed(ExactSize: 8);
164 break;
165 case RISCV::VL1RE8_V:
166 case RISCV::VL2RE8_V:
167 case RISCV::VL4RE8_V:
168 case RISCV::VL8RE8_V:
169 if (!MI.getOperand(i: 1).isFI())
170 return Register();
171 FrameIndex = MI.getOperand(i: 1).getIndex();
172 unsigned LMUL = *getLMULForRVVWholeLoadStore(Opcode: MI.getOpcode());
173 MemBytes = TypeSize::getScalable(MinimumSize: RISCV::RVVBytesPerBlock * LMUL);
174 return MI.getOperand(i: 0).getReg();
175 }
176
177 if (MI.getOperand(i: 1).isFI() && MI.getOperand(i: 2).isImm() &&
178 MI.getOperand(i: 2).getImm() == 0) {
179 FrameIndex = MI.getOperand(i: 1).getIndex();
180 return MI.getOperand(i: 0).getReg();
181 }
182
183 return 0;
184}
185
186Register RISCVInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
187 int &FrameIndex) const {
188 TypeSize Dummy = TypeSize::getZero();
189 return isStoreToStackSlot(MI, FrameIndex, MemBytes&: Dummy);
190}
191
192Register RISCVInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
193 int &FrameIndex,
194 TypeSize &MemBytes) const {
195 switch (MI.getOpcode()) {
196 default:
197 return 0;
198 case RISCV::SB:
199 MemBytes = TypeSize::getFixed(ExactSize: 1);
200 break;
201 case RISCV::SH:
202 case RISCV::SH_INX:
203 case RISCV::FSH:
204 MemBytes = TypeSize::getFixed(ExactSize: 2);
205 break;
206 case RISCV::SW:
207 case RISCV::SW_INX:
208 case RISCV::FSW:
209 MemBytes = TypeSize::getFixed(ExactSize: 4);
210 break;
211 case RISCV::SD:
212 case RISCV::SD_RV32:
213 case RISCV::FSD:
214 MemBytes = TypeSize::getFixed(ExactSize: 8);
215 break;
216 case RISCV::VS1R_V:
217 case RISCV::VS2R_V:
218 case RISCV::VS4R_V:
219 case RISCV::VS8R_V:
220 if (!MI.getOperand(i: 1).isFI())
221 return Register();
222 FrameIndex = MI.getOperand(i: 1).getIndex();
223 unsigned LMUL = *getLMULForRVVWholeLoadStore(Opcode: MI.getOpcode());
224 MemBytes = TypeSize::getScalable(MinimumSize: RISCV::RVVBytesPerBlock * LMUL);
225 return MI.getOperand(i: 0).getReg();
226 }
227
228 if (MI.getOperand(i: 1).isFI() && MI.getOperand(i: 2).isImm() &&
229 MI.getOperand(i: 2).getImm() == 0) {
230 FrameIndex = MI.getOperand(i: 1).getIndex();
231 return MI.getOperand(i: 0).getReg();
232 }
233
234 return 0;
235}
236
237bool RISCVInstrInfo::isReMaterializableImpl(
238 const MachineInstr &MI) const {
239 switch (RISCV::getRVVMCOpcode(RVVPseudoOpcode: MI.getOpcode())) {
240 case RISCV::VMV_V_X:
241 case RISCV::VFMV_V_F:
242 case RISCV::VMV_V_I:
243 case RISCV::VMV_S_X:
244 case RISCV::VFMV_S_F:
245 case RISCV::VID_V:
246 return MI.getOperand(i: 1).isUndef();
247 default:
248 return TargetInstrInfo::isReMaterializableImpl(MI);
249 }
250}
251
252static bool forwardCopyWillClobberTuple(unsigned DstReg, unsigned SrcReg,
253 unsigned NumRegs) {
254 return DstReg > SrcReg && (DstReg - SrcReg) < NumRegs;
255}
256
257static bool isConvertibleToVMV_V_V(const RISCVSubtarget &STI,
258 const MachineBasicBlock &MBB,
259 MachineBasicBlock::const_iterator MBBI,
260 MachineBasicBlock::const_iterator &DefMBBI,
261 RISCVVType::VLMUL LMul) {
262 if (PreferWholeRegisterMove)
263 return false;
264
265 assert(MBBI->getOpcode() == TargetOpcode::COPY &&
266 "Unexpected COPY instruction.");
267 Register SrcReg = MBBI->getOperand(i: 1).getReg();
268 const TargetRegisterInfo *TRI = STI.getRegisterInfo();
269
270 bool FoundDef = false;
271 bool FirstVSetVLI = false;
272 unsigned FirstSEW = 0;
273 while (MBBI != MBB.begin()) {
274 --MBBI;
275 if (MBBI->isMetaInstruction())
276 continue;
277
278 if (RISCVInstrInfo::isVectorConfigInstr(MI: *MBBI)) {
279 // There is a vsetvli between COPY and source define instruction.
280 // vy = def_vop ... (producing instruction)
281 // ...
282 // vsetvli
283 // ...
284 // vx = COPY vy
285 if (!FoundDef) {
286 if (!FirstVSetVLI) {
287 FirstVSetVLI = true;
288 unsigned FirstVType = MBBI->getOperand(i: 2).getImm();
289 RISCVVType::VLMUL FirstLMul = RISCVVType::getVLMUL(VType: FirstVType);
290 FirstSEW = RISCVVType::getSEW(VType: FirstVType);
291 // The first encountered vsetvli must have the same lmul as the
292 // register class of COPY.
293 if (FirstLMul != LMul)
294 return false;
295 }
296 // Only permit `vsetvli x0, x0, vtype` between COPY and the source
297 // define instruction.
298 if (!RISCVInstrInfo::isVLPreservingConfig(MI: *MBBI))
299 return false;
300 continue;
301 }
302
303 // MBBI is the first vsetvli before the producing instruction.
304 unsigned VType = MBBI->getOperand(i: 2).getImm();
305 // If there is a vsetvli between COPY and the producing instruction.
306 if (FirstVSetVLI) {
307 // If SEW is different, return false.
308 if (RISCVVType::getSEW(VType) != FirstSEW)
309 return false;
310 }
311
312 // If the vsetvli is tail undisturbed, keep the whole register move.
313 if (!RISCVVType::isTailAgnostic(VType))
314 return false;
315
316 // The checking is conservative. We only have register classes for
317 // LMUL = 1/2/4/8. We should be able to convert vmv1r.v to vmv.v.v
318 // for fractional LMUL operations. However, we could not use the vsetvli
319 // lmul for widening operations. The result of widening operation is
320 // 2 x LMUL.
321 return LMul == RISCVVType::getVLMUL(VType);
322 } else if (MBBI->isInlineAsm() || MBBI->isCall()) {
323 return false;
324 } else if (MBBI->getNumDefs()) {
325 // Check all the instructions which will change VL.
326 // For example, vleff has implicit def VL.
327 if (MBBI->modifiesRegister(Reg: RISCV::VL, /*TRI=*/nullptr))
328 return false;
329
330 // Only converting whole register copies to vmv.v.v when the defining
331 // value appears in the explicit operands.
332 for (const MachineOperand &MO : MBBI->explicit_operands()) {
333 if (!MO.isReg() || !MO.isDef())
334 continue;
335 if (!FoundDef && TRI->regsOverlap(RegA: MO.getReg(), RegB: SrcReg)) {
336 // We only permit the source of COPY has the same LMUL as the defined
337 // operand.
338 // There are cases we need to keep the whole register copy if the LMUL
339 // is different.
340 // For example,
341 // $x0 = PseudoVSETIVLI 4, 73 // vsetivli zero, 4, e16,m2,ta,m
342 // $v28m4 = PseudoVWADD_VV_M2 $v26m2, $v8m2
343 // # The COPY may be created by vlmul_trunc intrinsic.
344 // $v26m2 = COPY renamable $v28m2, implicit killed $v28m4
345 //
346 // After widening, the valid value will be 4 x e32 elements. If we
347 // convert the COPY to vmv.v.v, it will only copy 4 x e16 elements.
348 // FIXME: The COPY of subregister of Zvlsseg register will not be able
349 // to convert to vmv.v.[v|i] under the constraint.
350 if (MO.getReg() != SrcReg)
351 return false;
352
353 // In widening reduction instructions with LMUL_1 input vector case,
354 // only checking the LMUL is insufficient due to reduction result is
355 // always LMUL_1.
356 // For example,
357 // $x11 = PseudoVSETIVLI 1, 64 // vsetivli a1, 1, e8, m1, ta, mu
358 // $v8m1 = PseudoVWREDSUM_VS_M1 $v26, $v27
359 // $v26 = COPY killed renamable $v8
360 // After widening, The valid value will be 1 x e16 elements. If we
361 // convert the COPY to vmv.v.v, it will only copy 1 x e8 elements.
362 uint64_t TSFlags = MBBI->getDesc().TSFlags;
363 if (RISCVII::isRVVWideningReduction(TSFlags))
364 return false;
365
366 // If the producing instruction does not depend on vsetvli, do not
367 // convert COPY to vmv.v.v. For example, VL1R_V or PseudoVRELOAD.
368 if (!RISCVII::hasSEWOp(TSFlags) || !RISCVII::hasVLOp(TSFlags))
369 return false;
370
371 // Found the definition.
372 FoundDef = true;
373 DefMBBI = MBBI;
374 break;
375 }
376 }
377 }
378 }
379
380 return false;
381}
382
383void RISCVInstrInfo::copyPhysRegVector(
384 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
385 const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc,
386 const TargetRegisterClass *RegClass) const {
387 const RISCVRegisterInfo *TRI = STI.getRegisterInfo();
388 RISCVVType::VLMUL LMul = RISCVRI::getLMul(TSFlags: RegClass->TSFlags);
389 unsigned NF = RISCVRI::getNF(TSFlags: RegClass->TSFlags);
390
391 uint16_t SrcEncoding = TRI->getEncodingValue(Reg: SrcReg);
392 uint16_t DstEncoding = TRI->getEncodingValue(Reg: DstReg);
393 auto [LMulVal, Fractional] = RISCVVType::decodeVLMUL(VLMul: LMul);
394 assert(!Fractional && "It is impossible be fractional lmul here.");
395 unsigned NumRegs = NF * LMulVal;
396 bool ReversedCopy =
397 forwardCopyWillClobberTuple(DstReg: DstEncoding, SrcReg: SrcEncoding, NumRegs);
398 if (ReversedCopy) {
399 // If the src and dest overlap when copying a tuple, we need to copy the
400 // registers in reverse.
401 SrcEncoding += NumRegs - 1;
402 DstEncoding += NumRegs - 1;
403 }
404
405 unsigned I = 0;
406 auto GetCopyInfo = [&](uint16_t SrcEncoding, uint16_t DstEncoding)
407 -> std::tuple<RISCVVType::VLMUL, const TargetRegisterClass &, unsigned,
408 unsigned, unsigned> {
409 if (ReversedCopy) {
410 // For reversed copying, if there are enough aligned registers(8/4/2), we
411 // can do a larger copy(LMUL8/4/2).
412 // Besides, we have already known that DstEncoding is larger than
413 // SrcEncoding in forwardCopyWillClobberTuple, so the difference between
414 // DstEncoding and SrcEncoding should be >= LMUL value we try to use to
415 // avoid clobbering.
416 uint16_t Diff = DstEncoding - SrcEncoding;
417 if (I + 8 <= NumRegs && Diff >= 8 && SrcEncoding % 8 == 7 &&
418 DstEncoding % 8 == 7)
419 return {RISCVVType::LMUL_8, RISCV::VRM8RegClass, RISCV::VMV8R_V,
420 RISCV::PseudoVMV_V_V_M8, RISCV::PseudoVMV_V_I_M8};
421 if (I + 4 <= NumRegs && Diff >= 4 && SrcEncoding % 4 == 3 &&
422 DstEncoding % 4 == 3)
423 return {RISCVVType::LMUL_4, RISCV::VRM4RegClass, RISCV::VMV4R_V,
424 RISCV::PseudoVMV_V_V_M4, RISCV::PseudoVMV_V_I_M4};
425 if (I + 2 <= NumRegs && Diff >= 2 && SrcEncoding % 2 == 1 &&
426 DstEncoding % 2 == 1)
427 return {RISCVVType::LMUL_2, RISCV::VRM2RegClass, RISCV::VMV2R_V,
428 RISCV::PseudoVMV_V_V_M2, RISCV::PseudoVMV_V_I_M2};
429 // Or we should do LMUL1 copying.
430 return {RISCVVType::LMUL_1, RISCV::VRRegClass, RISCV::VMV1R_V,
431 RISCV::PseudoVMV_V_V_M1, RISCV::PseudoVMV_V_I_M1};
432 }
433
434 // For forward copying, if source register encoding and destination register
435 // encoding are aligned to 8/4/2, we can do a LMUL8/4/2 copying.
436 if (I + 8 <= NumRegs && SrcEncoding % 8 == 0 && DstEncoding % 8 == 0)
437 return {RISCVVType::LMUL_8, RISCV::VRM8RegClass, RISCV::VMV8R_V,
438 RISCV::PseudoVMV_V_V_M8, RISCV::PseudoVMV_V_I_M8};
439 if (I + 4 <= NumRegs && SrcEncoding % 4 == 0 && DstEncoding % 4 == 0)
440 return {RISCVVType::LMUL_4, RISCV::VRM4RegClass, RISCV::VMV4R_V,
441 RISCV::PseudoVMV_V_V_M4, RISCV::PseudoVMV_V_I_M4};
442 if (I + 2 <= NumRegs && SrcEncoding % 2 == 0 && DstEncoding % 2 == 0)
443 return {RISCVVType::LMUL_2, RISCV::VRM2RegClass, RISCV::VMV2R_V,
444 RISCV::PseudoVMV_V_V_M2, RISCV::PseudoVMV_V_I_M2};
445 // Or we should do LMUL1 copying.
446 return {RISCVVType::LMUL_1, RISCV::VRRegClass, RISCV::VMV1R_V,
447 RISCV::PseudoVMV_V_V_M1, RISCV::PseudoVMV_V_I_M1};
448 };
449
450 while (I != NumRegs) {
451 // For non-segment copying, we only do this once as the registers are always
452 // aligned.
453 // For segment copying, we may do this several times. If the registers are
454 // aligned to larger LMUL, we can eliminate some copyings.
455 auto [LMulCopied, RegClass, Opc, VVOpc, VIOpc] =
456 GetCopyInfo(SrcEncoding, DstEncoding);
457 auto [NumCopied, _] = RISCVVType::decodeVLMUL(VLMul: LMulCopied);
458
459 MachineBasicBlock::const_iterator DefMBBI;
460 if (LMul == LMulCopied &&
461 isConvertibleToVMV_V_V(STI, MBB, MBBI, DefMBBI, LMul)) {
462 Opc = VVOpc;
463 if (DefMBBI->getOpcode() == VIOpc)
464 Opc = VIOpc;
465 }
466
467 // Emit actual copying.
468 // For reversed copying, the encoding should be decreased.
469 MCRegister ActualSrcReg = TRI->findVRegWithEncoding(
470 RegClass, Encoding: ReversedCopy ? (SrcEncoding - NumCopied + 1) : SrcEncoding);
471 MCRegister ActualDstReg = TRI->findVRegWithEncoding(
472 RegClass, Encoding: ReversedCopy ? (DstEncoding - NumCopied + 1) : DstEncoding);
473
474 auto MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: Opc), DestReg: ActualDstReg);
475 bool UseVMV_V_I = RISCV::getRVVMCOpcode(RVVPseudoOpcode: Opc) == RISCV::VMV_V_I;
476 bool UseVMV = UseVMV_V_I || RISCV::getRVVMCOpcode(RVVPseudoOpcode: Opc) == RISCV::VMV_V_V;
477 if (UseVMV)
478 MIB.addReg(RegNo: ActualDstReg, Flags: RegState::Undef);
479 if (UseVMV_V_I)
480 MIB = MIB.add(MO: DefMBBI->getOperand(i: 2));
481 else
482 MIB = MIB.addReg(RegNo: ActualSrcReg, Flags: getKillRegState(B: KillSrc));
483 if (UseVMV) {
484 const MCInstrDesc &Desc = DefMBBI->getDesc();
485 MIB.add(MO: DefMBBI->getOperand(i: RISCVII::getVLOpNum(Desc))); // AVL
486 unsigned Log2SEW =
487 DefMBBI->getOperand(i: RISCVII::getSEWOpNum(Desc)).getImm();
488 MIB.addImm(Val: Log2SEW ? Log2SEW : 3); // SEW
489 MIB.addImm(Val: 0); // tu, mu
490 MIB.addReg(RegNo: RISCV::VL, Flags: RegState::Implicit);
491 MIB.addReg(RegNo: RISCV::VTYPE, Flags: RegState::Implicit);
492 }
493 // Add an implicit read of the original source to silence the verifier
494 // in the cases where some of the smaller VRs we're copying from might be
495 // undef, caused by the fact that the original, larger source VR might not
496 // be fully initialized at the time this COPY happens.
497 MIB.addReg(RegNo: SrcReg, Flags: RegState::Implicit);
498
499 // If we are copying reversely, we should decrease the encoding.
500 SrcEncoding += (ReversedCopy ? -NumCopied : NumCopied);
501 DstEncoding += (ReversedCopy ? -NumCopied : NumCopied);
502 I += NumCopied;
503 }
504}
505
506void RISCVInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
507 MachineBasicBlock::iterator MBBI,
508 const DebugLoc &DL, Register DstReg,
509 Register SrcReg, bool KillSrc,
510 bool RenamableDest, bool RenamableSrc) const {
511 const TargetRegisterInfo *TRI = STI.getRegisterInfo();
512 RegState KillFlag = getKillRegState(B: KillSrc);
513
514 if (RISCV::GPRRegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
515 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::ADDI), DestReg: DstReg)
516 .addReg(RegNo: SrcReg, Flags: KillFlag | getRenamableRegState(B: RenamableSrc))
517 .addImm(Val: 0);
518 return;
519 }
520
521 if (RISCV::GPRF16RegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
522 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::PseudoMV_FPR16INX), DestReg: DstReg)
523 .addReg(RegNo: SrcReg, Flags: KillFlag | getRenamableRegState(B: RenamableSrc));
524 return;
525 }
526
527 if (RISCV::GPRF32RegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
528 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::PseudoMV_FPR32INX), DestReg: DstReg)
529 .addReg(RegNo: SrcReg, Flags: KillFlag | getRenamableRegState(B: RenamableSrc));
530 return;
531 }
532
533 if (RISCV::GPRPairRegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
534 if (STI.isRV32() && STI.hasStdExtZdinx()) {
535 // On RV32_Zdinx, FMV.D will move a pair of registers to another pair of
536 // registers, in one instruction.
537 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FSGNJ_D_IN32X), DestReg: DstReg)
538 .addReg(RegNo: SrcReg, Flags: getRenamableRegState(B: RenamableSrc))
539 .addReg(RegNo: SrcReg, Flags: KillFlag | getRenamableRegState(B: RenamableSrc));
540 return;
541 }
542
543 MCRegister EvenReg = TRI->getSubReg(Reg: SrcReg, Idx: RISCV::sub_gpr_even);
544 MCRegister OddReg = TRI->getSubReg(Reg: SrcReg, Idx: RISCV::sub_gpr_odd);
545 // We need to correct the odd register of X0_Pair.
546 if (OddReg == RISCV::DUMMY_REG_PAIR_WITH_X0)
547 OddReg = RISCV::X0;
548 assert(DstReg != RISCV::X0_Pair && "Cannot write to X0_Pair");
549
550 // Emit an ADDI for both parts of GPRPair.
551 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::ADDI),
552 DestReg: TRI->getSubReg(Reg: DstReg, Idx: RISCV::sub_gpr_even))
553 .addReg(RegNo: EvenReg, Flags: KillFlag)
554 .addImm(Val: 0);
555 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::ADDI),
556 DestReg: TRI->getSubReg(Reg: DstReg, Idx: RISCV::sub_gpr_odd))
557 .addReg(RegNo: OddReg, Flags: KillFlag)
558 .addImm(Val: 0);
559 return;
560 }
561
562 // Handle copy from csr
563 if (RISCV::VCSRRegClass.contains(Reg: SrcReg) &&
564 RISCV::GPRRegClass.contains(Reg: DstReg)) {
565 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::CSRRS), DestReg: DstReg)
566 .addImm(Val: RISCVSysReg::lookupSysRegByName(Name: TRI->getName(RegNo: SrcReg))->Encoding)
567 .addReg(RegNo: RISCV::X0);
568 return;
569 }
570
571 if (RISCV::FPR16RegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
572 unsigned Opc;
573 if (STI.hasStdExtZfh()) {
574 Opc = RISCV::FSGNJ_H;
575 } else {
576 assert(STI.hasStdExtF() &&
577 (STI.hasStdExtZfhmin() || STI.hasStdExtZfbfmin()) &&
578 "Unexpected extensions");
579 // Zfhmin/Zfbfmin doesn't have FSGNJ_H, replace FSGNJ_H with FSGNJ_S.
580 DstReg = TRI->getMatchingSuperReg(Reg: DstReg, SubIdx: RISCV::sub_16,
581 RC: &RISCV::FPR32RegClass);
582 SrcReg = TRI->getMatchingSuperReg(Reg: SrcReg, SubIdx: RISCV::sub_16,
583 RC: &RISCV::FPR32RegClass);
584 Opc = RISCV::FSGNJ_S;
585 }
586 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: Opc), DestReg: DstReg)
587 .addReg(RegNo: SrcReg, Flags: KillFlag)
588 .addReg(RegNo: SrcReg, Flags: KillFlag);
589 return;
590 }
591
592 if (RISCV::FPR32RegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
593 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FSGNJ_S), DestReg: DstReg)
594 .addReg(RegNo: SrcReg, Flags: KillFlag)
595 .addReg(RegNo: SrcReg, Flags: KillFlag);
596 return;
597 }
598
599 if (RISCV::FPR64RegClass.contains(Reg1: DstReg, Reg2: SrcReg)) {
600 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FSGNJ_D), DestReg: DstReg)
601 .addReg(RegNo: SrcReg, Flags: KillFlag)
602 .addReg(RegNo: SrcReg, Flags: KillFlag);
603 return;
604 }
605
606 if (RISCV::FPR32RegClass.contains(Reg: DstReg) &&
607 RISCV::GPRRegClass.contains(Reg: SrcReg)) {
608 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FMV_W_X), DestReg: DstReg)
609 .addReg(RegNo: SrcReg, Flags: KillFlag);
610 return;
611 }
612
613 if (RISCV::GPRRegClass.contains(Reg: DstReg) &&
614 RISCV::FPR32RegClass.contains(Reg: SrcReg)) {
615 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FMV_X_W), DestReg: DstReg)
616 .addReg(RegNo: SrcReg, Flags: KillFlag);
617 return;
618 }
619
620 if (RISCV::FPR64RegClass.contains(Reg: DstReg) &&
621 RISCV::GPRRegClass.contains(Reg: SrcReg)) {
622 assert(STI.getXLen() == 64 && "Unexpected GPR size");
623 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FMV_D_X), DestReg: DstReg)
624 .addReg(RegNo: SrcReg, Flags: KillFlag);
625 return;
626 }
627
628 if (RISCV::GPRRegClass.contains(Reg: DstReg) &&
629 RISCV::FPR64RegClass.contains(Reg: SrcReg)) {
630 assert(STI.getXLen() == 64 && "Unexpected GPR size");
631 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: RISCV::FMV_X_D), DestReg: DstReg)
632 .addReg(RegNo: SrcReg, Flags: KillFlag);
633 return;
634 }
635
636 // VR->VR copies.
637 const TargetRegisterClass *RegClass =
638 TRI->getCommonMinimalPhysRegClass(Reg1: SrcReg, Reg2: DstReg);
639 if (RISCVRegisterInfo::isRVVRegClass(RC: RegClass)) {
640 copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RegClass);
641 return;
642 }
643
644 llvm_unreachable("Impossible reg-to-reg copy");
645}
646
647void RISCVInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
648 MachineBasicBlock::iterator I,
649 Register SrcReg, bool IsKill, int FI,
650 const TargetRegisterClass *RC,
651 Register VReg,
652 MachineInstr::MIFlag Flags) const {
653 MachineFunction *MF = MBB.getParent();
654 MachineFrameInfo &MFI = MF->getFrameInfo();
655 Align Alignment = MFI.getObjectAlign(ObjectIdx: FI);
656
657 unsigned Opcode;
658 if (RISCV::GPRRegClass.hasSubClassEq(RC)) {
659 Opcode = RegInfo.getRegSizeInBits(RC: RISCV::GPRRegClass) == 32 ? RISCV::SW
660 : RISCV::SD;
661 } else if (RISCV::GPRF16RegClass.hasSubClassEq(RC)) {
662 Opcode = RISCV::SH_INX;
663 } else if (RISCV::GPRF32RegClass.hasSubClassEq(RC)) {
664 Opcode = RISCV::SW_INX;
665 } else if (RISCV::GPRPairRegClass.hasSubClassEq(RC)) {
666 if (!STI.is64Bit() && STI.hasStdExtZilsd() &&
667 Alignment >= STI.getZilsdAlign()) {
668 Opcode = RISCV::SD_RV32;
669 } else {
670 Opcode = RISCV::PseudoRV32ZdinxSD;
671 }
672 } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) {
673 Opcode = RISCV::FSH;
674 } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) {
675 Opcode = RISCV::FSW;
676 } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) {
677 Opcode = RISCV::FSD;
678 } else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
679 Opcode = RISCV::VS1R_V;
680 } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
681 Opcode = RISCV::VS2R_V;
682 } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
683 Opcode = RISCV::VS4R_V;
684 } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
685 Opcode = RISCV::VS8R_V;
686 } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC))
687 Opcode = RISCV::PseudoVSPILL2_M1;
688 else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC))
689 Opcode = RISCV::PseudoVSPILL2_M2;
690 else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC))
691 Opcode = RISCV::PseudoVSPILL2_M4;
692 else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC))
693 Opcode = RISCV::PseudoVSPILL3_M1;
694 else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC))
695 Opcode = RISCV::PseudoVSPILL3_M2;
696 else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC))
697 Opcode = RISCV::PseudoVSPILL4_M1;
698 else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC))
699 Opcode = RISCV::PseudoVSPILL4_M2;
700 else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC))
701 Opcode = RISCV::PseudoVSPILL5_M1;
702 else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC))
703 Opcode = RISCV::PseudoVSPILL6_M1;
704 else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC))
705 Opcode = RISCV::PseudoVSPILL7_M1;
706 else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC))
707 Opcode = RISCV::PseudoVSPILL8_M1;
708 else
709 llvm_unreachable("Can't store this register to stack slot");
710
711 if (RISCVRegisterInfo::isRVVRegClass(RC)) {
712 MachineMemOperand *MMO = MF->getMachineMemOperand(
713 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), F: MachineMemOperand::MOStore,
714 Size: TypeSize::getScalable(MinimumSize: MFI.getObjectSize(ObjectIdx: FI)), BaseAlignment: Alignment);
715
716 MFI.setStackID(ObjectIdx: FI, ID: TargetStackID::ScalableVector);
717 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode))
718 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: IsKill))
719 .addFrameIndex(Idx: FI)
720 .addMemOperand(MMO)
721 .setMIFlag(Flags);
722 NumVRegSpilled += RegInfo.getRegSizeInBits(RC: *RC) / RISCV::RVVBitsPerBlock;
723 } else {
724 MachineMemOperand *MMO = MF->getMachineMemOperand(
725 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), F: MachineMemOperand::MOStore,
726 Size: MFI.getObjectSize(ObjectIdx: FI), BaseAlignment: Alignment);
727
728 BuildMI(BB&: MBB, I, MIMD: DebugLoc(), MCID: get(Opcode))
729 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: IsKill))
730 .addFrameIndex(Idx: FI)
731 .addImm(Val: 0)
732 .addMemOperand(MMO)
733 .setMIFlag(Flags);
734 }
735}
736
737void RISCVInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
738 MachineBasicBlock::iterator I,
739 Register DstReg, int FI,
740 const TargetRegisterClass *RC,
741 Register VReg, unsigned SubReg,
742 MachineInstr::MIFlag Flags) const {
743 MachineFunction *MF = MBB.getParent();
744 MachineFrameInfo &MFI = MF->getFrameInfo();
745 Align Alignment = MFI.getObjectAlign(ObjectIdx: FI);
746 DebugLoc DL =
747 Flags & MachineInstr::FrameDestroy ? MBB.findDebugLoc(MBBI: I) : DebugLoc();
748
749 unsigned Opcode;
750 if (RISCV::GPRRegClass.hasSubClassEq(RC)) {
751 Opcode = RegInfo.getRegSizeInBits(RC: RISCV::GPRRegClass) == 32 ? RISCV::LW
752 : RISCV::LD;
753 } else if (RISCV::GPRF16RegClass.hasSubClassEq(RC)) {
754 Opcode = RISCV::LH_INX;
755 } else if (RISCV::GPRF32RegClass.hasSubClassEq(RC)) {
756 Opcode = RISCV::LW_INX;
757 } else if (RISCV::GPRPairRegClass.hasSubClassEq(RC)) {
758 if (!STI.is64Bit() && STI.hasStdExtZilsd() &&
759 Alignment >= STI.getZilsdAlign()) {
760 Opcode = RISCV::LD_RV32;
761 } else {
762 Opcode = RISCV::PseudoRV32ZdinxLD;
763 }
764 } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) {
765 Opcode = RISCV::FLH;
766 } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) {
767 Opcode = RISCV::FLW;
768 } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) {
769 Opcode = RISCV::FLD;
770 } else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
771 Opcode = RISCV::VL1RE8_V;
772 } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
773 Opcode = RISCV::VL2RE8_V;
774 } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
775 Opcode = RISCV::VL4RE8_V;
776 } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
777 Opcode = RISCV::VL8RE8_V;
778 } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC))
779 Opcode = RISCV::PseudoVRELOAD2_M1;
780 else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC))
781 Opcode = RISCV::PseudoVRELOAD2_M2;
782 else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC))
783 Opcode = RISCV::PseudoVRELOAD2_M4;
784 else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC))
785 Opcode = RISCV::PseudoVRELOAD3_M1;
786 else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC))
787 Opcode = RISCV::PseudoVRELOAD3_M2;
788 else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC))
789 Opcode = RISCV::PseudoVRELOAD4_M1;
790 else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC))
791 Opcode = RISCV::PseudoVRELOAD4_M2;
792 else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC))
793 Opcode = RISCV::PseudoVRELOAD5_M1;
794 else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC))
795 Opcode = RISCV::PseudoVRELOAD6_M1;
796 else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC))
797 Opcode = RISCV::PseudoVRELOAD7_M1;
798 else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC))
799 Opcode = RISCV::PseudoVRELOAD8_M1;
800 else
801 llvm_unreachable("Can't load this register from stack slot");
802
803 if (RISCVRegisterInfo::isRVVRegClass(RC)) {
804 MachineMemOperand *MMO = MF->getMachineMemOperand(
805 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), F: MachineMemOperand::MOLoad,
806 Size: TypeSize::getScalable(MinimumSize: MFI.getObjectSize(ObjectIdx: FI)), BaseAlignment: Alignment);
807
808 MFI.setStackID(ObjectIdx: FI, ID: TargetStackID::ScalableVector);
809 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode), DestReg: DstReg)
810 .addFrameIndex(Idx: FI)
811 .addMemOperand(MMO)
812 .setMIFlag(Flags);
813 NumVRegReloaded += RegInfo.getRegSizeInBits(RC: *RC) / RISCV::RVVBitsPerBlock;
814 } else {
815 MachineMemOperand *MMO = MF->getMachineMemOperand(
816 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *MF, FI), F: MachineMemOperand::MOLoad,
817 Size: MFI.getObjectSize(ObjectIdx: FI), BaseAlignment: Alignment);
818
819 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode), DestReg: DstReg)
820 .addFrameIndex(Idx: FI)
821 .addImm(Val: 0)
822 .addMemOperand(MMO)
823 .setMIFlag(Flags);
824 }
825}
826std::optional<unsigned> getFoldedOpcode(MachineFunction &MF, MachineInstr &MI,
827 ArrayRef<unsigned> Ops,
828 const RISCVSubtarget &ST) {
829
830 // The below optimizations narrow the load so they are only valid for little
831 // endian.
832 // TODO: Support big endian by adding an offset into the frame object?
833 if (MF.getDataLayout().isBigEndian())
834 return std::nullopt;
835
836 // Fold load from stack followed by sext.b/sext.h/sext.w/zext.b/zext.h/zext.w.
837 if (Ops.size() != 1 || Ops[0] != 1)
838 return std::nullopt;
839
840 switch (MI.getOpcode()) {
841 default:
842 if (RISCVInstrInfo::isSEXT_W(MI))
843 return RISCV::LW;
844 if (RISCVInstrInfo::isZEXT_W(MI))
845 return RISCV::LWU;
846 if (RISCVInstrInfo::isZEXT_B(MI))
847 return RISCV::LBU;
848 break;
849 case RISCV::SEXT_H:
850 return RISCV::LH;
851 case RISCV::SEXT_B:
852 return RISCV::LB;
853 case RISCV::ZEXT_H_RV32:
854 case RISCV::ZEXT_H_RV64:
855 return RISCV::LHU;
856 }
857
858 switch (RISCV::getRVVMCOpcode(RVVPseudoOpcode: MI.getOpcode())) {
859 default:
860 return std::nullopt;
861 case RISCV::VMV_X_S: {
862 unsigned Log2SEW =
863 MI.getOperand(i: RISCVII::getSEWOpNum(Desc: MI.getDesc())).getImm();
864 if (ST.getXLen() < (1U << Log2SEW))
865 return std::nullopt;
866 switch (Log2SEW) {
867 case 3:
868 return RISCV::LB;
869 case 4:
870 return RISCV::LH;
871 case 5:
872 return RISCV::LW;
873 case 6:
874 return RISCV::LD;
875 default:
876 llvm_unreachable("Unexpected SEW");
877 }
878 }
879 case RISCV::VFMV_F_S: {
880 unsigned Log2SEW =
881 MI.getOperand(i: RISCVII::getSEWOpNum(Desc: MI.getDesc())).getImm();
882 switch (Log2SEW) {
883 case 4:
884 return RISCV::FLH;
885 case 5:
886 return RISCV::FLW;
887 case 6:
888 return RISCV::FLD;
889 default:
890 llvm_unreachable("Unexpected SEW");
891 }
892 }
893 }
894}
895
896// This is the version used during InlineSpiller::spillAroundUses
897MachineInstr *RISCVInstrInfo::foldMemoryOperandImpl(
898 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
899 MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS,
900 VirtRegMap *VRM) const {
901
902 std::optional<unsigned> LoadOpc = getFoldedOpcode(MF, MI, Ops, ST: STI);
903 if (!LoadOpc)
904 return nullptr;
905 Register DstReg = MI.getOperand(i: 0).getReg();
906 return BuildMI(BB&: *MI.getParent(), I: InsertPt, MIMD: MI.getDebugLoc(), MCID: get(Opcode: *LoadOpc),
907 DestReg: DstReg)
908 .addFrameIndex(Idx: FrameIndex)
909 .addImm(Val: 0);
910}
911
912static unsigned getLoadPredicatedOpcode(unsigned Opcode) {
913 switch (Opcode) {
914 case RISCV::LB:
915 return RISCV::PseudoCCLB;
916 case RISCV::LBU:
917 return RISCV::PseudoCCLBU;
918 case RISCV::LH:
919 return RISCV::PseudoCCLH;
920 case RISCV::LHU:
921 return RISCV::PseudoCCLHU;
922 case RISCV::LW:
923 return RISCV::PseudoCCLW;
924 case RISCV::LWU:
925 return RISCV::PseudoCCLWU;
926 case RISCV::LD:
927 return RISCV::PseudoCCLD;
928 case RISCV::QC_E_LB:
929 return RISCV::PseudoCCQC_E_LB;
930 case RISCV::QC_E_LBU:
931 return RISCV::PseudoCCQC_E_LBU;
932 case RISCV::QC_E_LH:
933 return RISCV::PseudoCCQC_E_LH;
934 case RISCV::QC_E_LHU:
935 return RISCV::PseudoCCQC_E_LHU;
936 case RISCV::QC_E_LW:
937 return RISCV::PseudoCCQC_E_LW;
938 default:
939 return 0;
940 }
941}
942
943MachineInstr *RISCVInstrInfo::foldMemoryOperandImpl(
944 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
945 MachineBasicBlock::iterator InsertPt, MachineInstr &LoadMI,
946 LiveIntervals *LIS) const {
947 // For now, only handle RISCV::PseudoCCMOVGPR.
948 if (MI.getOpcode() != RISCV::PseudoCCMOVGPR)
949 return nullptr;
950
951 unsigned PredOpc = getLoadPredicatedOpcode(Opcode: LoadMI.getOpcode());
952
953 if (!STI.hasShortForwardBranchILoad() || !PredOpc)
954 return nullptr;
955
956 MachineRegisterInfo &MRI = MF.getRegInfo();
957 if (Ops.size() != 1 || (Ops[0] != 4 && Ops[0] != 5))
958 return nullptr;
959
960 bool Invert = Ops[0] == 5;
961 const MachineOperand &FalseReg = MI.getOperand(i: !Invert ? 5 : 4);
962 Register DestReg = MI.getOperand(i: 0).getReg();
963 const TargetRegisterClass *PreviousClass = MRI.getRegClass(Reg: FalseReg.getReg());
964 if (!MRI.constrainRegClass(Reg: DestReg, RC: PreviousClass))
965 return nullptr;
966
967 // Create a new predicated version of DefMI.
968 MachineInstrBuilder NewMI = BuildMI(BB&: *MI.getParent(), I: InsertPt,
969 MIMD: MI.getDebugLoc(), MCID: get(Opcode: PredOpc), DestReg)
970 .add(MOs: {MI.getOperand(i: 1), MI.getOperand(i: 2)});
971
972 // Add condition code, inverting if necessary.
973 auto CC = static_cast<RISCVCC::CondCode>(MI.getOperand(i: 3).getImm());
974 if (!Invert)
975 CC = RISCVCC::getInverseBranchCondition(CC);
976 NewMI.addImm(Val: CC);
977
978 // Copy the false register.
979 NewMI.add(MO: FalseReg);
980
981 // Copy all the DefMI operands.
982 const MCInstrDesc &DefDesc = LoadMI.getDesc();
983 for (unsigned i = 1, e = DefDesc.getNumOperands(); i != e; ++i)
984 NewMI.add(MO: LoadMI.getOperand(i));
985
986 NewMI.cloneMemRefs(OtherMI: LoadMI);
987 return NewMI;
988}
989
990void RISCVInstrInfo::movImm(MachineBasicBlock &MBB,
991 MachineBasicBlock::iterator MBBI,
992 const DebugLoc &DL, Register DstReg, uint64_t Val,
993 MachineInstr::MIFlag Flag, bool DstRenamable,
994 bool DstIsDead) const {
995 Register SrcReg = RISCV::X0;
996
997 // For RV32, allow a sign or unsigned 32 bit value.
998 if (!STI.is64Bit() && !isInt<32>(x: Val)) {
999 // If have a uimm32 it will still fit in a register so we can allow it.
1000 if (!isUInt<32>(x: Val))
1001 report_fatal_error(reason: "Should only materialize 32-bit constants for RV32");
1002
1003 // Sign extend for generateInstSeq.
1004 Val = SignExtend64<32>(x: Val);
1005 }
1006
1007 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val, STI);
1008 assert(!Seq.empty());
1009
1010 bool SrcRenamable = false;
1011 unsigned Num = 0;
1012
1013 for (const RISCVMatInt::Inst &Inst : Seq) {
1014 bool LastItem = ++Num == Seq.size();
1015 RegState DstRegState = getDeadRegState(B: DstIsDead && LastItem) |
1016 getRenamableRegState(B: DstRenamable);
1017 RegState SrcRegState = getKillRegState(B: SrcReg != RISCV::X0) |
1018 getRenamableRegState(B: SrcRenamable);
1019 switch (Inst.getOpndKind()) {
1020 case RISCVMatInt::Imm:
1021 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: Inst.getOpcode()))
1022 .addReg(RegNo: DstReg, Flags: RegState::Define | DstRegState)
1023 .addImm(Val: Inst.getImm())
1024 .setMIFlag(Flag);
1025 break;
1026 case RISCVMatInt::RegX0:
1027 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: Inst.getOpcode()))
1028 .addReg(RegNo: DstReg, Flags: RegState::Define | DstRegState)
1029 .addReg(RegNo: SrcReg, Flags: SrcRegState)
1030 .addReg(RegNo: RISCV::X0)
1031 .setMIFlag(Flag);
1032 break;
1033 case RISCVMatInt::RegReg:
1034 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: Inst.getOpcode()))
1035 .addReg(RegNo: DstReg, Flags: RegState::Define | DstRegState)
1036 .addReg(RegNo: SrcReg, Flags: SrcRegState)
1037 .addReg(RegNo: SrcReg, Flags: SrcRegState)
1038 .setMIFlag(Flag);
1039 break;
1040 case RISCVMatInt::RegImm:
1041 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: get(Opcode: Inst.getOpcode()))
1042 .addReg(RegNo: DstReg, Flags: RegState::Define | DstRegState)
1043 .addReg(RegNo: SrcReg, Flags: SrcRegState)
1044 .addImm(Val: Inst.getImm())
1045 .setMIFlag(Flag);
1046 break;
1047 }
1048
1049 // Only the first instruction has X0 as its source.
1050 SrcReg = DstReg;
1051 SrcRenamable = DstRenamable;
1052 }
1053}
1054
1055RISCVCC::CondCode RISCVInstrInfo::getCondFromBranchOpc(unsigned Opc) {
1056 switch (Opc) {
1057 default:
1058 return RISCVCC::COND_INVALID;
1059 case RISCV::BEQ:
1060 case RISCV::BEQI:
1061 case RISCV::CV_BEQIMM:
1062 case RISCV::QC_BEQI:
1063 case RISCV::QC_E_BEQI:
1064 case RISCV::NDS_BBC:
1065 case RISCV::NDS_BEQC:
1066 return RISCVCC::COND_EQ;
1067 case RISCV::BNE:
1068 case RISCV::BNEI:
1069 case RISCV::QC_BNEI:
1070 case RISCV::QC_E_BNEI:
1071 case RISCV::CV_BNEIMM:
1072 case RISCV::NDS_BBS:
1073 case RISCV::NDS_BNEC:
1074 return RISCVCC::COND_NE;
1075 case RISCV::BLT:
1076 case RISCV::QC_BLTI:
1077 case RISCV::QC_E_BLTI:
1078 return RISCVCC::COND_LT;
1079 case RISCV::BGE:
1080 case RISCV::QC_BGEI:
1081 case RISCV::QC_E_BGEI:
1082 return RISCVCC::COND_GE;
1083 case RISCV::BLTU:
1084 case RISCV::QC_BLTUI:
1085 case RISCV::QC_E_BLTUI:
1086 return RISCVCC::COND_LTU;
1087 case RISCV::BGEU:
1088 case RISCV::QC_BGEUI:
1089 case RISCV::QC_E_BGEUI:
1090 return RISCVCC::COND_GEU;
1091 }
1092}
1093
1094bool RISCVInstrInfo::evaluateCondBranch(RISCVCC::CondCode CC, int64_t C0,
1095 int64_t C1) {
1096 switch (CC) {
1097 default:
1098 llvm_unreachable("Unexpected CC");
1099 case RISCVCC::COND_EQ:
1100 return C0 == C1;
1101 case RISCVCC::COND_NE:
1102 return C0 != C1;
1103 case RISCVCC::COND_LT:
1104 return C0 < C1;
1105 case RISCVCC::COND_GE:
1106 return C0 >= C1;
1107 case RISCVCC::COND_LTU:
1108 return (uint64_t)C0 < (uint64_t)C1;
1109 case RISCVCC::COND_GEU:
1110 return (uint64_t)C0 >= (uint64_t)C1;
1111 }
1112}
1113
1114// The contents of values added to Cond are not examined outside of
1115// RISCVInstrInfo, giving us flexibility in what to push to it. For RISCV, we
1116// push BranchOpcode, Reg1, Reg2.
1117static void parseCondBranch(MachineInstr &LastInst, MachineBasicBlock *&Target,
1118 SmallVectorImpl<MachineOperand> &Cond) {
1119 // Block ends with fall-through condbranch.
1120 assert(LastInst.getDesc().isConditionalBranch() &&
1121 "Unknown conditional branch");
1122 Target = LastInst.getOperand(i: 2).getMBB();
1123 Cond.push_back(Elt: MachineOperand::CreateImm(Val: LastInst.getOpcode()));
1124 Cond.push_back(Elt: LastInst.getOperand(i: 0));
1125 Cond.push_back(Elt: LastInst.getOperand(i: 1));
1126}
1127
1128static unsigned getInverseXqcicmOpcode(unsigned Opcode) {
1129 switch (Opcode) {
1130 default:
1131 llvm_unreachable("Unexpected Opcode");
1132 case RISCV::QC_MVEQ:
1133 return RISCV::QC_MVNE;
1134 case RISCV::QC_MVNE:
1135 return RISCV::QC_MVEQ;
1136 case RISCV::QC_MVLT:
1137 return RISCV::QC_MVGE;
1138 case RISCV::QC_MVGE:
1139 return RISCV::QC_MVLT;
1140 case RISCV::QC_MVLTU:
1141 return RISCV::QC_MVGEU;
1142 case RISCV::QC_MVGEU:
1143 return RISCV::QC_MVLTU;
1144 case RISCV::QC_MVEQI:
1145 return RISCV::QC_MVNEI;
1146 case RISCV::QC_MVNEI:
1147 return RISCV::QC_MVEQI;
1148 case RISCV::QC_MVLTI:
1149 return RISCV::QC_MVGEI;
1150 case RISCV::QC_MVGEI:
1151 return RISCV::QC_MVLTI;
1152 case RISCV::QC_MVLTUI:
1153 return RISCV::QC_MVGEUI;
1154 case RISCV::QC_MVGEUI:
1155 return RISCV::QC_MVLTUI;
1156 }
1157}
1158
1159unsigned RISCVCC::getBrCond(RISCVCC::CondCode CC, unsigned SelectOpc) {
1160 switch (SelectOpc) {
1161 default:
1162 switch (CC) {
1163 default:
1164 llvm_unreachable("Unexpected condition code!");
1165 case RISCVCC::COND_EQ:
1166 return RISCV::BEQ;
1167 case RISCVCC::COND_NE:
1168 return RISCV::BNE;
1169 case RISCVCC::COND_LT:
1170 return RISCV::BLT;
1171 case RISCVCC::COND_GE:
1172 return RISCV::BGE;
1173 case RISCVCC::COND_LTU:
1174 return RISCV::BLTU;
1175 case RISCVCC::COND_GEU:
1176 return RISCV::BGEU;
1177 }
1178 break;
1179 case RISCV::Select_GPR_Using_CC_Imm5_Zibi:
1180 switch (CC) {
1181 default:
1182 llvm_unreachable("Unexpected condition code!");
1183 case RISCVCC::COND_EQ:
1184 return RISCV::BEQI;
1185 case RISCVCC::COND_NE:
1186 return RISCV::BNEI;
1187 }
1188 break;
1189 case RISCV::Select_GPR_Using_CC_SImm5_CV:
1190 switch (CC) {
1191 default:
1192 llvm_unreachable("Unexpected condition code!");
1193 case RISCVCC::COND_EQ:
1194 return RISCV::CV_BEQIMM;
1195 case RISCVCC::COND_NE:
1196 return RISCV::CV_BNEIMM;
1197 }
1198 break;
1199 case RISCV::Select_GPRNoX0_Using_CC_SImm5NonZero_QC:
1200 switch (CC) {
1201 default:
1202 llvm_unreachable("Unexpected condition code!");
1203 case RISCVCC::COND_EQ:
1204 return RISCV::QC_BEQI;
1205 case RISCVCC::COND_NE:
1206 return RISCV::QC_BNEI;
1207 case RISCVCC::COND_LT:
1208 return RISCV::QC_BLTI;
1209 case RISCVCC::COND_GE:
1210 return RISCV::QC_BGEI;
1211 }
1212 break;
1213 case RISCV::Select_GPRNoX0_Using_CC_UImm5NonZero_QC:
1214 switch (CC) {
1215 default:
1216 llvm_unreachable("Unexpected condition code!");
1217 case RISCVCC::COND_LTU:
1218 return RISCV::QC_BLTUI;
1219 case RISCVCC::COND_GEU:
1220 return RISCV::QC_BGEUI;
1221 }
1222 break;
1223 case RISCV::Select_GPRNoX0_Using_CC_SImm16NonZero_QC:
1224 switch (CC) {
1225 default:
1226 llvm_unreachable("Unexpected condition code!");
1227 case RISCVCC::COND_EQ:
1228 return RISCV::QC_E_BEQI;
1229 case RISCVCC::COND_NE:
1230 return RISCV::QC_E_BNEI;
1231 case RISCVCC::COND_LT:
1232 return RISCV::QC_E_BLTI;
1233 case RISCVCC::COND_GE:
1234 return RISCV::QC_E_BGEI;
1235 }
1236 break;
1237 case RISCV::Select_GPRNoX0_Using_CC_UImm16NonZero_QC:
1238 switch (CC) {
1239 default:
1240 llvm_unreachable("Unexpected condition code!");
1241 case RISCVCC::COND_LTU:
1242 return RISCV::QC_E_BLTUI;
1243 case RISCVCC::COND_GEU:
1244 return RISCV::QC_E_BGEUI;
1245 }
1246 break;
1247 case RISCV::Select_GPR_Using_CC_UImmLog2XLen_NDS:
1248 switch (CC) {
1249 default:
1250 llvm_unreachable("Unexpected condition code!");
1251 case RISCVCC::COND_EQ:
1252 return RISCV::NDS_BBC;
1253 case RISCVCC::COND_NE:
1254 return RISCV::NDS_BBS;
1255 }
1256 break;
1257 case RISCV::Select_GPR_Using_CC_UImm7_NDS:
1258 switch (CC) {
1259 default:
1260 llvm_unreachable("Unexpected condition code!");
1261 case RISCVCC::COND_EQ:
1262 return RISCV::NDS_BEQC;
1263 case RISCVCC::COND_NE:
1264 return RISCV::NDS_BNEC;
1265 }
1266 break;
1267 }
1268}
1269
1270RISCVCC::CondCode RISCVCC::getInverseBranchCondition(RISCVCC::CondCode CC) {
1271 switch (CC) {
1272 default:
1273 llvm_unreachable("Unrecognized conditional branch");
1274 case RISCVCC::COND_EQ:
1275 return RISCVCC::COND_NE;
1276 case RISCVCC::COND_NE:
1277 return RISCVCC::COND_EQ;
1278 case RISCVCC::COND_LT:
1279 return RISCVCC::COND_GE;
1280 case RISCVCC::COND_GE:
1281 return RISCVCC::COND_LT;
1282 case RISCVCC::COND_LTU:
1283 return RISCVCC::COND_GEU;
1284 case RISCVCC::COND_GEU:
1285 return RISCVCC::COND_LTU;
1286 }
1287}
1288
1289bool RISCVInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
1290 MachineBasicBlock *&TBB,
1291 MachineBasicBlock *&FBB,
1292 SmallVectorImpl<MachineOperand> &Cond,
1293 bool AllowModify) const {
1294 TBB = FBB = nullptr;
1295 Cond.clear();
1296
1297 // If the block has no terminators, it just falls into the block after it.
1298 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
1299 if (I == MBB.end() || !isUnpredicatedTerminator(MI: *I))
1300 return false;
1301
1302 // Count the number of terminators and find the first unconditional or
1303 // indirect branch.
1304 MachineBasicBlock::iterator FirstUncondOrIndirectBr = MBB.end();
1305 int NumTerminators = 0;
1306 for (auto J = I.getReverse(); J != MBB.rend() && isUnpredicatedTerminator(MI: *J);
1307 J++) {
1308 NumTerminators++;
1309 if (J->getDesc().isUnconditionalBranch() ||
1310 J->getDesc().isIndirectBranch()) {
1311 FirstUncondOrIndirectBr = J.getReverse();
1312 }
1313 }
1314
1315 // If AllowModify is true, we can erase any terminators after
1316 // FirstUncondOrIndirectBR.
1317 if (AllowModify && FirstUncondOrIndirectBr != MBB.end()) {
1318 while (std::next(x: FirstUncondOrIndirectBr) != MBB.end()) {
1319 std::next(x: FirstUncondOrIndirectBr)->eraseFromParent();
1320 NumTerminators--;
1321 }
1322 I = FirstUncondOrIndirectBr;
1323 }
1324
1325 // We can't handle blocks that end in an indirect branch.
1326 if (I->getDesc().isIndirectBranch())
1327 return true;
1328
1329 // We can't handle Generic branch opcodes from Global ISel.
1330 if (I->isPreISelOpcode())
1331 return true;
1332
1333 // We can't handle blocks with more than 2 terminators.
1334 if (NumTerminators > 2)
1335 return true;
1336
1337 // Handle a single unconditional branch.
1338 if (NumTerminators == 1 && I->getDesc().isUnconditionalBranch()) {
1339 TBB = getBranchDestBlock(MI: *I);
1340 return false;
1341 }
1342
1343 // Handle a single conditional branch.
1344 if (NumTerminators == 1 && I->getDesc().isConditionalBranch()) {
1345 parseCondBranch(LastInst&: *I, Target&: TBB, Cond);
1346 return false;
1347 }
1348
1349 // Handle a conditional branch followed by an unconditional branch.
1350 if (NumTerminators == 2 && std::prev(x: I)->getDesc().isConditionalBranch() &&
1351 I->getDesc().isUnconditionalBranch()) {
1352 parseCondBranch(LastInst&: *std::prev(x: I), Target&: TBB, Cond);
1353 FBB = getBranchDestBlock(MI: *I);
1354 return false;
1355 }
1356
1357 // Otherwise, we can't handle this.
1358 return true;
1359}
1360
1361unsigned RISCVInstrInfo::removeBranch(MachineBasicBlock &MBB,
1362 int *BytesRemoved) const {
1363 if (BytesRemoved)
1364 *BytesRemoved = 0;
1365 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
1366 if (I == MBB.end())
1367 return 0;
1368
1369 if (!I->getDesc().isUnconditionalBranch() &&
1370 !I->getDesc().isConditionalBranch())
1371 return 0;
1372
1373 // Remove the branch.
1374 if (BytesRemoved)
1375 *BytesRemoved += getInstSizeInBytes(MI: *I);
1376 I->eraseFromParent();
1377
1378 I = MBB.end();
1379
1380 if (I == MBB.begin())
1381 return 1;
1382 --I;
1383 if (!I->getDesc().isConditionalBranch())
1384 return 1;
1385
1386 // Remove the branch.
1387 if (BytesRemoved)
1388 *BytesRemoved += getInstSizeInBytes(MI: *I);
1389 I->eraseFromParent();
1390 return 2;
1391}
1392
1393// Inserts a branch into the end of the specific MachineBasicBlock, returning
1394// the number of instructions inserted.
1395unsigned RISCVInstrInfo::insertBranch(
1396 MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
1397 ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const {
1398 if (BytesAdded)
1399 *BytesAdded = 0;
1400
1401 // Shouldn't be a fall through.
1402 assert(TBB && "insertBranch must not be told to insert a fallthrough");
1403 assert((Cond.size() == 3 || Cond.size() == 0) &&
1404 "RISC-V branch conditions have two components!");
1405
1406 // Unconditional branch.
1407 if (Cond.empty()) {
1408 MachineInstr &MI = *BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: RISCV::PseudoBR)).addMBB(MBB: TBB);
1409 if (BytesAdded)
1410 *BytesAdded += getInstSizeInBytes(MI);
1411 return 1;
1412 }
1413
1414 // Either a one or two-way conditional branch.
1415 MachineInstr &CondMI = *BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: Cond[0].getImm()))
1416 .add(MO: Cond[1])
1417 .add(MO: Cond[2])
1418 .addMBB(MBB: TBB);
1419 if (BytesAdded)
1420 *BytesAdded += getInstSizeInBytes(MI: CondMI);
1421
1422 // One-way conditional branch.
1423 if (!FBB)
1424 return 1;
1425
1426 // Two-way conditional branch.
1427 MachineInstr &MI = *BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: RISCV::PseudoBR)).addMBB(MBB: FBB);
1428 if (BytesAdded)
1429 *BytesAdded += getInstSizeInBytes(MI);
1430 return 2;
1431}
1432
1433void RISCVInstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
1434 MachineBasicBlock &DestBB,
1435 MachineBasicBlock &RestoreBB,
1436 const DebugLoc &DL, int64_t BrOffset,
1437 RegScavenger *RS) const {
1438 assert(RS && "RegScavenger required for long branching");
1439 assert(MBB.empty() &&
1440 "new block should be inserted for expanding unconditional branch");
1441 assert(MBB.pred_size() == 1);
1442 assert(RestoreBB.empty() &&
1443 "restore block should be inserted for restoring clobbered registers");
1444
1445 MachineFunction *MF = MBB.getParent();
1446 MachineRegisterInfo &MRI = MF->getRegInfo();
1447 RISCVMachineFunctionInfo *RVFI = MF->getInfo<RISCVMachineFunctionInfo>();
1448 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
1449
1450 if (!isInt<32>(x: BrOffset))
1451 report_fatal_error(
1452 reason: "Branch offsets outside of the signed 32-bit range not supported");
1453
1454 // FIXME: A virtual register must be used initially, as the register
1455 // scavenger won't work with empty blocks (SIInstrInfo::insertIndirectBranch
1456 // uses the same workaround).
1457 Register ScratchReg = MRI.createVirtualRegister(RegClass: &RISCV::GPRJALRRegClass);
1458 auto II = MBB.end();
1459 // We may also update the jump target to RestoreBB later.
1460 MachineInstr &MI = *BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::PseudoJump))
1461 .addReg(RegNo: ScratchReg, Flags: RegState::Define | RegState::Dead)
1462 .addMBB(MBB: &DestBB, TargetFlags: RISCVII::MO_CALL);
1463
1464 RS->enterBasicBlockEnd(MBB);
1465 const TargetRegisterClass *RC = &RISCV::GPRRegClass;
1466 if (STI.hasStdExtZicfilp())
1467 RC = &RISCV::GPRX7RegClass;
1468 Register TmpGPR =
1469 RS->scavengeRegisterBackwards(RC: *RC, To: MI.getIterator(),
1470 /*RestoreAfter=*/false, /*SpAdj=*/SPAdj: 0,
1471 /*AllowSpill=*/false);
1472 if (TmpGPR.isValid())
1473 RS->setRegUsed(Reg: TmpGPR);
1474 else {
1475 // The case when there is no scavenged register needs special handling.
1476
1477 // Pick s11(or s1 for rve) because it doesn't make a difference.
1478 TmpGPR = STI.hasStdExtE() ? RISCV::X9 : RISCV::X27;
1479 // Force t2 if Zicfilp is on
1480 if (STI.hasStdExtZicfilp())
1481 TmpGPR = RISCV::X7;
1482
1483 int FrameIndex = RVFI->getBranchRelaxationScratchFrameIndex();
1484 if (FrameIndex == -1)
1485 report_fatal_error(reason: "underestimated function size");
1486
1487 storeRegToStackSlot(MBB, I: MI, SrcReg: TmpGPR, /*IsKill=*/true, FI: FrameIndex,
1488 RC: &RISCV::GPRRegClass, VReg: Register());
1489 TRI->eliminateFrameIndex(MI: std::prev(x: MI.getIterator()),
1490 /*SpAdj=*/SPAdj: 0, /*FIOperandNum=*/1);
1491
1492 MI.getOperand(i: 1).setMBB(&RestoreBB);
1493
1494 loadRegFromStackSlot(MBB&: RestoreBB, I: RestoreBB.end(), DstReg: TmpGPR, FI: FrameIndex,
1495 RC: &RISCV::GPRRegClass, VReg: Register());
1496 TRI->eliminateFrameIndex(MI: RestoreBB.back(),
1497 /*SpAdj=*/SPAdj: 0, /*FIOperandNum=*/1);
1498 }
1499
1500 MRI.replaceRegWith(FromReg: ScratchReg, ToReg: TmpGPR);
1501 MRI.clearVirtRegs();
1502}
1503
1504bool RISCVInstrInfo::reverseBranchCondition(
1505 SmallVectorImpl<MachineOperand> &Cond) const {
1506 assert((Cond.size() == 3) && "Invalid branch condition!");
1507 switch (Cond[0].getImm()) {
1508 default:
1509 llvm_unreachable("Unknown conditional branch!");
1510 case RISCV::BEQ:
1511 Cond[0].setImm(RISCV::BNE);
1512 break;
1513 case RISCV::BEQI:
1514 Cond[0].setImm(RISCV::BNEI);
1515 break;
1516 case RISCV::BNE:
1517 Cond[0].setImm(RISCV::BEQ);
1518 break;
1519 case RISCV::BNEI:
1520 Cond[0].setImm(RISCV::BEQI);
1521 break;
1522 case RISCV::BLT:
1523 Cond[0].setImm(RISCV::BGE);
1524 break;
1525 case RISCV::BGE:
1526 Cond[0].setImm(RISCV::BLT);
1527 break;
1528 case RISCV::BLTU:
1529 Cond[0].setImm(RISCV::BGEU);
1530 break;
1531 case RISCV::BGEU:
1532 Cond[0].setImm(RISCV::BLTU);
1533 break;
1534 case RISCV::CV_BEQIMM:
1535 Cond[0].setImm(RISCV::CV_BNEIMM);
1536 break;
1537 case RISCV::CV_BNEIMM:
1538 Cond[0].setImm(RISCV::CV_BEQIMM);
1539 break;
1540 case RISCV::QC_BEQI:
1541 Cond[0].setImm(RISCV::QC_BNEI);
1542 break;
1543 case RISCV::QC_BNEI:
1544 Cond[0].setImm(RISCV::QC_BEQI);
1545 break;
1546 case RISCV::QC_BGEI:
1547 Cond[0].setImm(RISCV::QC_BLTI);
1548 break;
1549 case RISCV::QC_BLTI:
1550 Cond[0].setImm(RISCV::QC_BGEI);
1551 break;
1552 case RISCV::QC_BGEUI:
1553 Cond[0].setImm(RISCV::QC_BLTUI);
1554 break;
1555 case RISCV::QC_BLTUI:
1556 Cond[0].setImm(RISCV::QC_BGEUI);
1557 break;
1558 case RISCV::QC_E_BEQI:
1559 Cond[0].setImm(RISCV::QC_E_BNEI);
1560 break;
1561 case RISCV::QC_E_BNEI:
1562 Cond[0].setImm(RISCV::QC_E_BEQI);
1563 break;
1564 case RISCV::QC_E_BGEI:
1565 Cond[0].setImm(RISCV::QC_E_BLTI);
1566 break;
1567 case RISCV::QC_E_BLTI:
1568 Cond[0].setImm(RISCV::QC_E_BGEI);
1569 break;
1570 case RISCV::QC_E_BGEUI:
1571 Cond[0].setImm(RISCV::QC_E_BLTUI);
1572 break;
1573 case RISCV::QC_E_BLTUI:
1574 Cond[0].setImm(RISCV::QC_E_BGEUI);
1575 break;
1576 case RISCV::NDS_BBC:
1577 Cond[0].setImm(RISCV::NDS_BBS);
1578 break;
1579 case RISCV::NDS_BBS:
1580 Cond[0].setImm(RISCV::NDS_BBC);
1581 break;
1582 case RISCV::NDS_BEQC:
1583 Cond[0].setImm(RISCV::NDS_BNEC);
1584 break;
1585 case RISCV::NDS_BNEC:
1586 Cond[0].setImm(RISCV::NDS_BEQC);
1587 break;
1588 }
1589
1590 return false;
1591}
1592
1593// Return true if the instruction is a load immediate instruction (i.e.
1594// ADDI x0, imm).
1595static bool isLoadImm(const MachineInstr *MI, int64_t &Imm) {
1596 if (MI->getOpcode() == RISCV::ADDI && MI->getOperand(i: 1).isReg() &&
1597 MI->getOperand(i: 1).getReg() == RISCV::X0) {
1598 Imm = MI->getOperand(i: 2).getImm();
1599 return true;
1600 }
1601 return false;
1602}
1603
1604bool RISCVInstrInfo::isFromLoadImm(const MachineRegisterInfo &MRI,
1605 const MachineOperand &Op, int64_t &Imm) {
1606 // Either a load from immediate instruction or X0.
1607 if (!Op.isReg())
1608 return false;
1609
1610 Register Reg = Op.getReg();
1611 if (Reg == RISCV::X0) {
1612 Imm = 0;
1613 return true;
1614 }
1615 return Reg.isVirtual() && isLoadImm(MI: MRI.getVRegDef(Reg), Imm);
1616}
1617
1618bool RISCVInstrInfo::optimizeCondBranch(MachineInstr &MI) const {
1619 bool IsSigned = false;
1620 bool IsEquality = false;
1621 switch (MI.getOpcode()) {
1622 default:
1623 return false;
1624 case RISCV::BEQ:
1625 case RISCV::BNE:
1626 IsEquality = true;
1627 break;
1628 case RISCV::BGE:
1629 case RISCV::BLT:
1630 IsSigned = true;
1631 break;
1632 case RISCV::BGEU:
1633 case RISCV::BLTU:
1634 break;
1635 }
1636
1637 MachineBasicBlock *MBB = MI.getParent();
1638 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
1639
1640 const MachineOperand &LHS = MI.getOperand(i: 0);
1641 const MachineOperand &RHS = MI.getOperand(i: 1);
1642 MachineBasicBlock *TBB = MI.getOperand(i: 2).getMBB();
1643
1644 RISCVCC::CondCode CC = getCondFromBranchOpc(Opc: MI.getOpcode());
1645 assert(CC != RISCVCC::COND_INVALID);
1646
1647 // Canonicalize conditional branches which can be constant folded into
1648 // beqz or bnez. We can't modify the CFG here.
1649 int64_t C0, C1;
1650 if (isFromLoadImm(MRI, Op: LHS, Imm&: C0) && isFromLoadImm(MRI, Op: RHS, Imm&: C1)) {
1651 unsigned NewOpc = evaluateCondBranch(CC, C0, C1) ? RISCV::BEQ : RISCV::BNE;
1652 // Build the new branch and remove the old one.
1653 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
1654 .addReg(RegNo: RISCV::X0)
1655 .addReg(RegNo: RISCV::X0)
1656 .addMBB(MBB: TBB);
1657 MI.eraseFromParent();
1658 return true;
1659 }
1660
1661 if (IsEquality)
1662 return false;
1663
1664 // For two constants C0 and C1 from
1665 // ```
1666 // li Y, C0
1667 // li Z, C1
1668 // ```
1669 // 1. if C1 = C0 + 1
1670 // we can turn:
1671 // (a) blt Y, X -> bge X, Z
1672 // (b) bge Y, X -> blt X, Z
1673 //
1674 // 2. if C1 = C0 - 1
1675 // we can turn:
1676 // (a) blt X, Y -> bge Z, X
1677 // (b) bge X, Y -> blt Z, X
1678 //
1679 // To make sure this optimization is really beneficial, we only
1680 // optimize for cases where Y had only one use (i.e. only used by the branch).
1681 // Try to find the register for constant Z; return
1682 // invalid register otherwise.
1683 auto searchConst = [&](int64_t C1) -> Register {
1684 MachineBasicBlock::reverse_iterator II(&MI), E = MBB->rend();
1685 auto DefC1 = std::find_if(first: ++II, last: E, pred: [&](const MachineInstr &I) -> bool {
1686 int64_t Imm;
1687 return isLoadImm(MI: &I, Imm) && Imm == C1 &&
1688 I.getOperand(i: 0).getReg().isVirtual();
1689 });
1690 if (DefC1 != E)
1691 return DefC1->getOperand(i: 0).getReg();
1692
1693 return Register();
1694 };
1695
1696 unsigned NewOpc = RISCVCC::getBrCond(CC: getInverseBranchCondition(CC));
1697
1698 // Might be case 1.
1699 // Don't change 0 to 1 since we can use x0.
1700 // For unsigned cases changing -1U to 0 would be incorrect.
1701 // The incorrect case for signed would be INT_MAX, but isFromLoadImm can't
1702 // return that.
1703 if (isFromLoadImm(MRI, Op: LHS, Imm&: C0) && C0 != 0 && LHS.getReg().isVirtual() &&
1704 MRI.hasOneUse(RegNo: LHS.getReg()) && (IsSigned || C0 != -1)) {
1705 assert(isInt<12>(C0) && "Unexpected immediate");
1706 if (Register RegZ = searchConst(C0 + 1)) {
1707 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
1708 .add(MO: RHS)
1709 .addReg(RegNo: RegZ)
1710 .addMBB(MBB: TBB);
1711 // We might extend the live range of Z, clear its kill flag to
1712 // account for this.
1713 MRI.clearKillFlags(Reg: RegZ);
1714 MI.eraseFromParent();
1715 return true;
1716 }
1717 }
1718
1719 // Might be case 2.
1720 // For signed cases we don't want to change 0 since we can use x0.
1721 // For unsigned cases changing 0 to -1U would be incorrect.
1722 // The incorrect case for signed would be INT_MIN, but isFromLoadImm can't
1723 // return that.
1724 if (isFromLoadImm(MRI, Op: RHS, Imm&: C0) && C0 != 0 && RHS.getReg().isVirtual() &&
1725 MRI.hasOneUse(RegNo: RHS.getReg())) {
1726 assert(isInt<12>(C0) && "Unexpected immediate");
1727 if (Register RegZ = searchConst(C0 - 1)) {
1728 BuildMI(BB&: *MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
1729 .addReg(RegNo: RegZ)
1730 .add(MO: LHS)
1731 .addMBB(MBB: TBB);
1732 // We might extend the live range of Z, clear its kill flag to
1733 // account for this.
1734 MRI.clearKillFlags(Reg: RegZ);
1735 MI.eraseFromParent();
1736 return true;
1737 }
1738 }
1739
1740 return false;
1741}
1742
1743MachineBasicBlock *
1744RISCVInstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
1745 assert(MI.getDesc().isBranch() && "Unexpected opcode!");
1746 // The branch target is always the last operand.
1747 int NumOp = MI.getNumExplicitOperands();
1748 return MI.getOperand(i: NumOp - 1).getMBB();
1749}
1750
1751bool RISCVInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
1752 int64_t BrOffset) const {
1753 unsigned XLen = STI.getXLen();
1754 // Ideally we could determine the supported branch offset from the
1755 // RISCVII::FormMask, but this can't be used for Pseudo instructions like
1756 // PseudoBR.
1757 switch (BranchOp) {
1758 default:
1759 llvm_unreachable("Unexpected opcode!");
1760 case RISCV::NDS_BBC:
1761 case RISCV::NDS_BBS:
1762 case RISCV::NDS_BEQC:
1763 case RISCV::NDS_BNEC:
1764 return isInt<11>(x: BrOffset);
1765 case RISCV::BEQ:
1766 case RISCV::BNE:
1767 case RISCV::BLT:
1768 case RISCV::BGE:
1769 case RISCV::BLTU:
1770 case RISCV::BGEU:
1771 case RISCV::BEQI:
1772 case RISCV::BNEI:
1773 case RISCV::CV_BEQIMM:
1774 case RISCV::CV_BNEIMM:
1775 case RISCV::QC_BEQI:
1776 case RISCV::QC_BNEI:
1777 case RISCV::QC_BGEI:
1778 case RISCV::QC_BLTI:
1779 case RISCV::QC_BLTUI:
1780 case RISCV::QC_BGEUI:
1781 case RISCV::QC_E_BEQI:
1782 case RISCV::QC_E_BNEI:
1783 case RISCV::QC_E_BGEI:
1784 case RISCV::QC_E_BLTI:
1785 case RISCV::QC_E_BLTUI:
1786 case RISCV::QC_E_BGEUI:
1787 return isInt<13>(x: BrOffset);
1788 case RISCV::JAL:
1789 case RISCV::PseudoBR:
1790 return isInt<21>(x: BrOffset);
1791 case RISCV::PseudoJump:
1792 return isInt<32>(x: SignExtend64(X: BrOffset + 0x800, B: XLen));
1793 }
1794}
1795
1796// If the operation has a predicated pseudo instruction, return the pseudo
1797// instruction opcode. Otherwise, return RISCV::INSTRUCTION_LIST_END.
1798// TODO: Support more operations.
1799unsigned getPredicatedOpcode(unsigned Opcode) {
1800 // clang-format off
1801 switch (Opcode) {
1802 case RISCV::ADD: return RISCV::PseudoCCADD;
1803 case RISCV::SUB: return RISCV::PseudoCCSUB;
1804 case RISCV::SLL: return RISCV::PseudoCCSLL;
1805 case RISCV::SRL: return RISCV::PseudoCCSRL;
1806 case RISCV::SRA: return RISCV::PseudoCCSRA;
1807 case RISCV::AND: return RISCV::PseudoCCAND;
1808 case RISCV::OR: return RISCV::PseudoCCOR;
1809 case RISCV::XOR: return RISCV::PseudoCCXOR;
1810 case RISCV::MAX: return RISCV::PseudoCCMAX;
1811 case RISCV::MAXU: return RISCV::PseudoCCMAXU;
1812 case RISCV::MIN: return RISCV::PseudoCCMIN;
1813 case RISCV::MINU: return RISCV::PseudoCCMINU;
1814 case RISCV::MUL: return RISCV::PseudoCCMUL;
1815 case RISCV::LUI: return RISCV::PseudoCCLUI;
1816 case RISCV::QC_LI: return RISCV::PseudoCCQC_LI;
1817 case RISCV::QC_E_LI: return RISCV::PseudoCCQC_E_LI;
1818
1819 case RISCV::ADDI: return RISCV::PseudoCCADDI;
1820 case RISCV::SLLI: return RISCV::PseudoCCSLLI;
1821 case RISCV::SRLI: return RISCV::PseudoCCSRLI;
1822 case RISCV::SRAI: return RISCV::PseudoCCSRAI;
1823 case RISCV::ANDI: return RISCV::PseudoCCANDI;
1824 case RISCV::ORI: return RISCV::PseudoCCORI;
1825 case RISCV::XORI: return RISCV::PseudoCCXORI;
1826
1827 case RISCV::ADDW: return RISCV::PseudoCCADDW;
1828 case RISCV::SUBW: return RISCV::PseudoCCSUBW;
1829 case RISCV::SLLW: return RISCV::PseudoCCSLLW;
1830 case RISCV::SRLW: return RISCV::PseudoCCSRLW;
1831 case RISCV::SRAW: return RISCV::PseudoCCSRAW;
1832
1833 case RISCV::ADDIW: return RISCV::PseudoCCADDIW;
1834 case RISCV::SLLIW: return RISCV::PseudoCCSLLIW;
1835 case RISCV::SRLIW: return RISCV::PseudoCCSRLIW;
1836 case RISCV::SRAIW: return RISCV::PseudoCCSRAIW;
1837
1838 case RISCV::ANDN: return RISCV::PseudoCCANDN;
1839 case RISCV::ORN: return RISCV::PseudoCCORN;
1840 case RISCV::XNOR: return RISCV::PseudoCCXNOR;
1841
1842 case RISCV::NDS_BFOS: return RISCV::PseudoCCNDS_BFOS;
1843 case RISCV::NDS_BFOZ: return RISCV::PseudoCCNDS_BFOZ;
1844 }
1845 // clang-format on
1846
1847 return RISCV::INSTRUCTION_LIST_END;
1848}
1849
1850/// Identify instructions that can be folded into a CCMOV instruction, and
1851/// return the defining instruction.
1852static MachineInstr *canFoldAsPredicatedOp(Register Reg,
1853 const MachineRegisterInfo &MRI,
1854 const TargetInstrInfo *TII,
1855 const RISCVSubtarget &STI) {
1856 if (!Reg.isVirtual())
1857 return nullptr;
1858 if (!MRI.hasOneNonDBGUse(RegNo: Reg))
1859 return nullptr;
1860 MachineInstr *MI = MRI.getVRegDef(Reg);
1861 if (!MI)
1862 return nullptr;
1863
1864 if (!STI.hasShortForwardBranchIMinMax() &&
1865 (MI->getOpcode() == RISCV::MAX || MI->getOpcode() == RISCV::MIN ||
1866 MI->getOpcode() == RISCV::MINU || MI->getOpcode() == RISCV::MAXU))
1867 return nullptr;
1868
1869 if (!STI.hasShortForwardBranchIMul() && MI->getOpcode() == RISCV::MUL)
1870 return nullptr;
1871
1872 // Check if MI can be predicated and folded into the CCMOV.
1873 if (getPredicatedOpcode(Opcode: MI->getOpcode()) == RISCV::INSTRUCTION_LIST_END)
1874 return nullptr;
1875 // Don't predicate li idiom.
1876 if (MI->getOpcode() == RISCV::ADDI && MI->getOperand(i: 1).isReg() &&
1877 MI->getOperand(i: 1).getReg() == RISCV::X0)
1878 return nullptr;
1879 // Check if MI has any other defs or physreg uses.
1880 for (const MachineOperand &MO : llvm::drop_begin(RangeOrContainer: MI->operands())) {
1881 // Reject frame index operands, PEI can't handle the predicated pseudos.
1882 if (MO.isFI() || MO.isCPI() || MO.isJTI())
1883 return nullptr;
1884 if (!MO.isReg())
1885 continue;
1886 // MI can't have any tied operands, that would conflict with predication.
1887 if (MO.isTied())
1888 return nullptr;
1889 if (MO.isDef())
1890 return nullptr;
1891 // Allow constant physregs.
1892 if (MO.getReg().isPhysical() && !MRI.isConstantPhysReg(PhysReg: MO.getReg()))
1893 return nullptr;
1894 }
1895 bool DontMoveAcrossStores = true;
1896 if (!MI->isSafeToMove(SawStore&: DontMoveAcrossStores))
1897 return nullptr;
1898 return MI;
1899}
1900
1901bool RISCVInstrInfo::analyzeSelect(const MachineInstr &MI,
1902 SmallVectorImpl<MachineOperand> &Cond,
1903 unsigned &TrueOp, unsigned &FalseOp,
1904 bool &Optimizable) const {
1905 assert(MI.getOpcode() == RISCV::PseudoCCMOVGPR &&
1906 "Unknown select instruction");
1907 // CCMOV operands:
1908 // 0: Def.
1909 // 1: LHS of compare.
1910 // 2: RHS of compare.
1911 // 3: Condition code.
1912 // 4: False use.
1913 // 5: True use.
1914 TrueOp = 5;
1915 FalseOp = 4;
1916 Cond.push_back(Elt: MI.getOperand(i: 1));
1917 Cond.push_back(Elt: MI.getOperand(i: 2));
1918 Cond.push_back(Elt: MI.getOperand(i: 3));
1919 // We can only fold when we support short forward branch opt.
1920 Optimizable = STI.hasShortForwardBranchIALU();
1921 return false;
1922}
1923
1924MachineInstr *
1925RISCVInstrInfo::optimizeSelect(MachineInstr &MI,
1926 SmallPtrSetImpl<MachineInstr *> &SeenMIs,
1927 bool PreferFalse) const {
1928 assert(MI.getOpcode() == RISCV::PseudoCCMOVGPR &&
1929 "Unknown select instruction");
1930 if (!STI.hasShortForwardBranchIALU())
1931 return nullptr;
1932
1933 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
1934 MachineInstr *DefMI =
1935 canFoldAsPredicatedOp(Reg: MI.getOperand(i: 5).getReg(), MRI, TII: this, STI);
1936 bool Invert = !DefMI;
1937 if (!DefMI)
1938 DefMI = canFoldAsPredicatedOp(Reg: MI.getOperand(i: 4).getReg(), MRI, TII: this, STI);
1939 if (!DefMI)
1940 return nullptr;
1941
1942 // Find new register class to use.
1943 MachineOperand FalseReg = MI.getOperand(i: Invert ? 5 : 4);
1944 Register DestReg = MI.getOperand(i: 0).getReg();
1945 const TargetRegisterClass *PreviousClass = MRI.getRegClass(Reg: FalseReg.getReg());
1946 if (!MRI.constrainRegClass(Reg: DestReg, RC: PreviousClass))
1947 return nullptr;
1948
1949 unsigned PredOpc = getPredicatedOpcode(Opcode: DefMI->getOpcode());
1950 assert(PredOpc != RISCV::INSTRUCTION_LIST_END && "Unexpected opcode!");
1951
1952 // Create a new predicated version of DefMI.
1953 MachineInstrBuilder NewMI =
1954 BuildMI(BB&: *MI.getParent(), I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: PredOpc), DestReg);
1955
1956 // Copy the condition portion.
1957 NewMI.add(MO: MI.getOperand(i: 1));
1958 NewMI.add(MO: MI.getOperand(i: 2));
1959
1960 // Add condition code, inverting if necessary.
1961 auto CC = static_cast<RISCVCC::CondCode>(MI.getOperand(i: 3).getImm());
1962 if (Invert)
1963 CC = RISCVCC::getInverseBranchCondition(CC);
1964 NewMI.addImm(Val: CC);
1965
1966 // Copy the false register.
1967 NewMI.add(MO: FalseReg);
1968
1969 // Copy all the DefMI operands.
1970 const MCInstrDesc &DefDesc = DefMI->getDesc();
1971 for (unsigned i = 1, e = DefDesc.getNumOperands(); i != e; ++i)
1972 NewMI.add(MO: DefMI->getOperand(i));
1973
1974 // Update SeenMIs set: register newly created MI and erase removed DefMI.
1975 SeenMIs.insert(Ptr: NewMI);
1976 SeenMIs.erase(Ptr: DefMI);
1977
1978 // If MI is inside a loop, and DefMI is outside the loop, then kill flags on
1979 // DefMI would be invalid when transferred inside the loop. Checking for a
1980 // loop is expensive, but at least remove kill flags if they are in different
1981 // BBs.
1982 if (DefMI->getParent() != MI.getParent())
1983 NewMI->clearKillInfo();
1984
1985 // The caller will erase MI, but not DefMI.
1986 DefMI->eraseFromParent();
1987 return NewMI;
1988}
1989
1990unsigned RISCVInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
1991 if (MI.isMetaInstruction())
1992 return 0;
1993
1994 unsigned Opcode = MI.getOpcode();
1995
1996 if (Opcode == TargetOpcode::INLINEASM ||
1997 Opcode == TargetOpcode::INLINEASM_BR) {
1998 const MachineFunction &MF = *MI.getParent()->getParent();
1999 return getInlineAsmLength(Str: MI.getOperand(i: 0).getSymbolName(),
2000 MAI: *MF.getTarget().getMCAsmInfo());
2001 }
2002
2003 if (!MI.memoperands_empty()) {
2004 MachineMemOperand *MMO = *(MI.memoperands_begin());
2005 if (STI.hasStdExtZihintntl() && MMO->isNonTemporal()) {
2006 if (STI.hasStdExtZca()) {
2007 if (isCompressibleInst(MI, STI))
2008 return 4; // c.ntl.all + c.load/c.store
2009 return 6; // c.ntl.all + load/store
2010 }
2011 return 8; // ntl.all + load/store
2012 }
2013 }
2014
2015 if (Opcode == TargetOpcode::BUNDLE)
2016 return getInstBundleLength(MI);
2017
2018 if (MI.getParent() && MI.getParent()->getParent()) {
2019 if (isCompressibleInst(MI, STI))
2020 return 2;
2021 }
2022
2023 switch (Opcode) {
2024 case RISCV::PseudoMV_FPR16INX:
2025 case RISCV::PseudoMV_FPR32INX:
2026 // MV is always compressible to either c.mv or c.li rd, 0.
2027 return STI.hasStdExtZca() ? 2 : 4;
2028 case TargetOpcode::STACKMAP:
2029 // The upper bound for a stackmap intrinsic is the full length of its shadow
2030 return StackMapOpers(&MI).getNumPatchBytes();
2031 case TargetOpcode::PATCHPOINT:
2032 // The size of the patchpoint intrinsic is the number of bytes requested
2033 return PatchPointOpers(&MI).getNumPatchBytes();
2034 case TargetOpcode::STATEPOINT: {
2035 // The size of the statepoint intrinsic is the number of bytes requested
2036 unsigned NumBytes = StatepointOpers(&MI).getNumPatchBytes();
2037 // No patch bytes means at most a PseudoCall is emitted
2038 return std::max(a: NumBytes, b: 8U);
2039 }
2040 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
2041 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
2042 case TargetOpcode::PATCHABLE_TAIL_CALL: {
2043 const MachineFunction &MF = *MI.getParent()->getParent();
2044 const Function &F = MF.getFunction();
2045 if (Opcode == TargetOpcode::PATCHABLE_FUNCTION_ENTER &&
2046 F.hasFnAttribute(Kind: "patchable-function-entry")) {
2047 unsigned Num;
2048 if (F.getFnAttribute(Kind: "patchable-function-entry")
2049 .getValueAsString()
2050 .getAsInteger(Radix: 10, Result&: Num))
2051 return get(Opcode).getSize();
2052
2053 // Number of C.NOP or NOP
2054 return (STI.hasStdExtZca() ? 2 : 4) * Num;
2055 }
2056 // XRay uses C.JAL + 21 or 33 C.NOP for each sled in RV32 and RV64,
2057 // respectively.
2058 return STI.is64Bit() ? 68 : 44;
2059 }
2060 default:
2061 return get(Opcode).getSize();
2062 }
2063}
2064
2065unsigned RISCVInstrInfo::getInstBundleLength(const MachineInstr &MI) const {
2066 unsigned Size = 0;
2067 MachineBasicBlock::const_instr_iterator I = MI.getIterator();
2068 MachineBasicBlock::const_instr_iterator E = MI.getParent()->instr_end();
2069 while (++I != E && I->isInsideBundle()) {
2070 assert(!I->isBundle() && "No nested bundle!");
2071 Size += getInstSizeInBytes(MI: *I);
2072 }
2073 return Size;
2074}
2075
2076bool RISCVInstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const {
2077 const unsigned Opcode = MI.getOpcode();
2078 switch (Opcode) {
2079 default:
2080 break;
2081 case RISCV::FSGNJ_D:
2082 case RISCV::FSGNJ_S:
2083 case RISCV::FSGNJ_H:
2084 case RISCV::FSGNJ_D_INX:
2085 case RISCV::FSGNJ_D_IN32X:
2086 case RISCV::FSGNJ_S_INX:
2087 case RISCV::FSGNJ_H_INX:
2088 // The canonical floating-point move is fsgnj rd, rs, rs.
2089 return MI.getOperand(i: 1).isReg() && MI.getOperand(i: 2).isReg() &&
2090 MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg();
2091 case RISCV::ADDI:
2092 case RISCV::ORI:
2093 case RISCV::XORI:
2094 return (MI.getOperand(i: 1).isReg() &&
2095 MI.getOperand(i: 1).getReg() == RISCV::X0) ||
2096 (MI.getOperand(i: 2).isImm() && MI.getOperand(i: 2).getImm() == 0);
2097 }
2098 return MI.isAsCheapAsAMove();
2099}
2100
2101std::optional<DestSourcePair>
2102RISCVInstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
2103 if (MI.isMoveReg())
2104 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2105 switch (MI.getOpcode()) {
2106 default:
2107 break;
2108 case RISCV::ADD:
2109 case RISCV::OR:
2110 case RISCV::XOR:
2111 if (MI.getOperand(i: 1).isReg() && MI.getOperand(i: 1).getReg() == RISCV::X0 &&
2112 MI.getOperand(i: 2).isReg())
2113 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 2)};
2114 if (MI.getOperand(i: 2).isReg() && MI.getOperand(i: 2).getReg() == RISCV::X0 &&
2115 MI.getOperand(i: 1).isReg())
2116 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2117 break;
2118 case RISCV::ADDI:
2119 // Operand 1 can be a frameindex but callers expect registers
2120 if (MI.getOperand(i: 1).isReg() && MI.getOperand(i: 2).isImm() &&
2121 MI.getOperand(i: 2).getImm() == 0)
2122 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2123 break;
2124 case RISCV::SUB:
2125 if (MI.getOperand(i: 2).isReg() && MI.getOperand(i: 2).getReg() == RISCV::X0 &&
2126 MI.getOperand(i: 1).isReg())
2127 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2128 break;
2129 case RISCV::SH1ADD:
2130 case RISCV::SH1ADD_UW:
2131 case RISCV::SH2ADD:
2132 case RISCV::SH2ADD_UW:
2133 case RISCV::SH3ADD:
2134 case RISCV::SH3ADD_UW:
2135 if (MI.getOperand(i: 1).isReg() && MI.getOperand(i: 1).getReg() == RISCV::X0 &&
2136 MI.getOperand(i: 2).isReg())
2137 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 2)};
2138 break;
2139 case RISCV::FSGNJ_D:
2140 case RISCV::FSGNJ_S:
2141 case RISCV::FSGNJ_H:
2142 case RISCV::FSGNJ_D_INX:
2143 case RISCV::FSGNJ_D_IN32X:
2144 case RISCV::FSGNJ_S_INX:
2145 case RISCV::FSGNJ_H_INX:
2146 // The canonical floating-point move is fsgnj rd, rs, rs.
2147 if (MI.getOperand(i: 1).isReg() && MI.getOperand(i: 2).isReg() &&
2148 MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg())
2149 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 1)};
2150 break;
2151 }
2152 return std::nullopt;
2153}
2154
2155MachineTraceStrategy RISCVInstrInfo::getMachineCombinerTraceStrategy() const {
2156 if (ForceMachineCombinerStrategy.getNumOccurrences() == 0) {
2157 // The option is unused. Choose Local strategy only for in-order cores. When
2158 // scheduling model is unspecified, use MinInstrCount strategy as more
2159 // generic one.
2160 const auto &SchedModel = STI.getSchedModel();
2161 return (!SchedModel.hasInstrSchedModel() || SchedModel.isOutOfOrder())
2162 ? MachineTraceStrategy::TS_MinInstrCount
2163 : MachineTraceStrategy::TS_Local;
2164 }
2165 // The strategy was forced by the option.
2166 return ForceMachineCombinerStrategy;
2167}
2168
2169void RISCVInstrInfo::finalizeInsInstrs(
2170 MachineInstr &Root, unsigned &Pattern,
2171 SmallVectorImpl<MachineInstr *> &InsInstrs) const {
2172 int16_t FrmOpIdx =
2173 RISCV::getNamedOperandIdx(Opcode: Root.getOpcode(), Name: RISCV::OpName::frm);
2174 if (FrmOpIdx < 0) {
2175 assert(all_of(InsInstrs,
2176 [](MachineInstr *MI) {
2177 return RISCV::getNamedOperandIdx(MI->getOpcode(),
2178 RISCV::OpName::frm) < 0;
2179 }) &&
2180 "New instructions require FRM whereas the old one does not have it");
2181 return;
2182 }
2183
2184 const MachineOperand &FRM = Root.getOperand(i: FrmOpIdx);
2185 MachineFunction &MF = *Root.getMF();
2186
2187 for (auto *NewMI : InsInstrs) {
2188 // We'd already added the FRM operand.
2189 if (static_cast<unsigned>(RISCV::getNamedOperandIdx(
2190 Opcode: NewMI->getOpcode(), Name: RISCV::OpName::frm)) != NewMI->getNumOperands())
2191 continue;
2192 MachineInstrBuilder MIB(MF, NewMI);
2193 MIB.add(MO: FRM);
2194 if (FRM.getImm() == RISCVFPRndMode::DYN)
2195 MIB.addUse(RegNo: RISCV::FRM, Flags: RegState::Implicit);
2196 }
2197}
2198
2199static bool isFADD(unsigned Opc) {
2200 switch (Opc) {
2201 default:
2202 return false;
2203 case RISCV::FADD_H:
2204 case RISCV::FADD_S:
2205 case RISCV::FADD_D:
2206 return true;
2207 }
2208}
2209
2210static bool isFSUB(unsigned Opc) {
2211 switch (Opc) {
2212 default:
2213 return false;
2214 case RISCV::FSUB_H:
2215 case RISCV::FSUB_S:
2216 case RISCV::FSUB_D:
2217 return true;
2218 }
2219}
2220
2221static bool isFMUL(unsigned Opc) {
2222 switch (Opc) {
2223 default:
2224 return false;
2225 case RISCV::FMUL_H:
2226 case RISCV::FMUL_S:
2227 case RISCV::FMUL_D:
2228 return true;
2229 }
2230}
2231
2232bool RISCVInstrInfo::isVectorAssociativeAndCommutative(const MachineInstr &Inst,
2233 bool Invert) const {
2234#define OPCODE_LMUL_CASE(OPC) \
2235 case RISCV::OPC##_M1: \
2236 case RISCV::OPC##_M2: \
2237 case RISCV::OPC##_M4: \
2238 case RISCV::OPC##_M8: \
2239 case RISCV::OPC##_MF2: \
2240 case RISCV::OPC##_MF4: \
2241 case RISCV::OPC##_MF8
2242
2243#define OPCODE_LMUL_MASK_CASE(OPC) \
2244 case RISCV::OPC##_M1_MASK: \
2245 case RISCV::OPC##_M2_MASK: \
2246 case RISCV::OPC##_M4_MASK: \
2247 case RISCV::OPC##_M8_MASK: \
2248 case RISCV::OPC##_MF2_MASK: \
2249 case RISCV::OPC##_MF4_MASK: \
2250 case RISCV::OPC##_MF8_MASK
2251
2252 unsigned Opcode = Inst.getOpcode();
2253 if (Invert) {
2254 if (auto InvOpcode = getInverseOpcode(Opcode))
2255 Opcode = *InvOpcode;
2256 else
2257 return false;
2258 }
2259
2260 // clang-format off
2261 switch (Opcode) {
2262 default:
2263 return false;
2264 OPCODE_LMUL_CASE(PseudoVADD_VV):
2265 OPCODE_LMUL_MASK_CASE(PseudoVADD_VV):
2266 OPCODE_LMUL_CASE(PseudoVMUL_VV):
2267 OPCODE_LMUL_MASK_CASE(PseudoVMUL_VV):
2268 return true;
2269 }
2270 // clang-format on
2271
2272#undef OPCODE_LMUL_MASK_CASE
2273#undef OPCODE_LMUL_CASE
2274}
2275
2276bool RISCVInstrInfo::areRVVInstsReassociable(const MachineInstr &Root,
2277 const MachineInstr &Prev) const {
2278 if (!areOpcodesEqualOrInverse(Opcode1: Root.getOpcode(), Opcode2: Prev.getOpcode()))
2279 return false;
2280
2281 assert(Root.getMF() == Prev.getMF());
2282 const MachineRegisterInfo *MRI = &Root.getMF()->getRegInfo();
2283 const TargetRegisterInfo *TRI = MRI->getTargetRegisterInfo();
2284
2285 // Make sure vtype operands are also the same.
2286 const MCInstrDesc &Desc = get(Opcode: Root.getOpcode());
2287 const uint64_t TSFlags = Desc.TSFlags;
2288
2289 auto checkImmOperand = [&](unsigned OpIdx) {
2290 return Root.getOperand(i: OpIdx).getImm() == Prev.getOperand(i: OpIdx).getImm();
2291 };
2292
2293 auto checkRegOperand = [&](unsigned OpIdx) {
2294 return Root.getOperand(i: OpIdx).getReg() == Prev.getOperand(i: OpIdx).getReg();
2295 };
2296
2297 // PassThru
2298 // TODO: Potentially we can loosen the condition to consider Root to be
2299 // associable with Prev if Root has NoReg as passthru. In which case we
2300 // also need to loosen the condition on vector policies between these.
2301 if (!checkRegOperand(1))
2302 return false;
2303
2304 // SEW
2305 if (RISCVII::hasSEWOp(TSFlags) &&
2306 !checkImmOperand(RISCVII::getSEWOpNum(Desc)))
2307 return false;
2308
2309 // Mask
2310 if (RISCVII::usesMaskPolicy(TSFlags)) {
2311 const MachineBasicBlock *MBB = Root.getParent();
2312 const MachineBasicBlock::const_reverse_iterator It1(&Root);
2313 const MachineBasicBlock::const_reverse_iterator It2(&Prev);
2314 Register MI1VReg;
2315
2316 bool SeenMI2 = false;
2317 for (auto End = MBB->rend(), It = It1; It != End; ++It) {
2318 if (It == It2) {
2319 SeenMI2 = true;
2320 if (!MI1VReg.isValid())
2321 // There is no V0 def between Root and Prev; they're sharing the
2322 // same V0.
2323 break;
2324 }
2325
2326 if (It->modifiesRegister(Reg: RISCV::V0, TRI)) {
2327 Register SrcReg = It->getOperand(i: 1).getReg();
2328 // If it's not VReg it'll be more difficult to track its defs, so
2329 // bailing out here just to be safe.
2330 if (!SrcReg.isVirtual())
2331 return false;
2332
2333 if (!MI1VReg.isValid()) {
2334 // This is the V0 def for Root.
2335 MI1VReg = SrcReg;
2336 continue;
2337 }
2338
2339 // Some random mask updates.
2340 if (!SeenMI2)
2341 continue;
2342
2343 // This is the V0 def for Prev; check if it's the same as that of
2344 // Root.
2345 if (MI1VReg != SrcReg)
2346 return false;
2347 else
2348 break;
2349 }
2350 }
2351
2352 // If we haven't encountered Prev, it's likely that this function was
2353 // called in a wrong way (e.g. Root is before Prev).
2354 assert(SeenMI2 && "Prev is expected to appear before Root");
2355 }
2356
2357 // Tail / Mask policies
2358 if (RISCVII::hasVecPolicyOp(TSFlags) &&
2359 !checkImmOperand(RISCVII::getVecPolicyOpNum(Desc)))
2360 return false;
2361
2362 // VL
2363 if (RISCVII::hasVLOp(TSFlags)) {
2364 unsigned OpIdx = RISCVII::getVLOpNum(Desc);
2365 const MachineOperand &Op1 = Root.getOperand(i: OpIdx);
2366 const MachineOperand &Op2 = Prev.getOperand(i: OpIdx);
2367 if (Op1.getType() != Op2.getType())
2368 return false;
2369 switch (Op1.getType()) {
2370 case MachineOperand::MO_Register:
2371 if (Op1.getReg() != Op2.getReg())
2372 return false;
2373 break;
2374 case MachineOperand::MO_Immediate:
2375 if (Op1.getImm() != Op2.getImm())
2376 return false;
2377 break;
2378 default:
2379 llvm_unreachable("Unrecognized VL operand type");
2380 }
2381 }
2382
2383 // Rounding modes
2384 if (RISCVII::hasRoundModeOp(TSFlags) &&
2385 !checkImmOperand(RISCVII::getVLOpNum(Desc) - 1))
2386 return false;
2387
2388 return true;
2389}
2390
2391// Most of our RVV pseudos have passthru operand, so the real operands
2392// start from index = 2.
2393bool RISCVInstrInfo::hasReassociableVectorSibling(const MachineInstr &Inst,
2394 bool &Commuted) const {
2395 const MachineBasicBlock *MBB = Inst.getParent();
2396 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
2397 assert(RISCVII::isFirstDefTiedToFirstUse(get(Inst.getOpcode())) &&
2398 "Expect the present of passthrough operand.");
2399 MachineInstr *MI1 = MRI.getUniqueVRegDef(Reg: Inst.getOperand(i: 2).getReg());
2400 MachineInstr *MI2 = MRI.getUniqueVRegDef(Reg: Inst.getOperand(i: 3).getReg());
2401
2402 // If only one operand has the same or inverse opcode and it's the second
2403 // source operand, the operands must be commuted.
2404 Commuted = !areRVVInstsReassociable(Root: Inst, Prev: *MI1) &&
2405 areRVVInstsReassociable(Root: Inst, Prev: *MI2);
2406 if (Commuted)
2407 std::swap(a&: MI1, b&: MI2);
2408
2409 return areRVVInstsReassociable(Root: Inst, Prev: *MI1) &&
2410 (isVectorAssociativeAndCommutative(Inst: *MI1) ||
2411 isVectorAssociativeAndCommutative(Inst: *MI1, /* Invert */ true)) &&
2412 hasReassociableOperands(Inst: *MI1, MBB) &&
2413 MRI.hasOneNonDBGUse(RegNo: MI1->getOperand(i: 0).getReg());
2414}
2415
2416bool RISCVInstrInfo::hasReassociableOperands(
2417 const MachineInstr &Inst, const MachineBasicBlock *MBB) const {
2418 if (!isVectorAssociativeAndCommutative(Inst) &&
2419 !isVectorAssociativeAndCommutative(Inst, /*Invert=*/true))
2420 return TargetInstrInfo::hasReassociableOperands(Inst, MBB);
2421
2422 const MachineOperand &Op1 = Inst.getOperand(i: 2);
2423 const MachineOperand &Op2 = Inst.getOperand(i: 3);
2424 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
2425
2426 // We need virtual register definitions for the operands that we will
2427 // reassociate.
2428 MachineInstr *MI1 = nullptr;
2429 MachineInstr *MI2 = nullptr;
2430 if (Op1.isReg() && Op1.getReg().isVirtual())
2431 MI1 = MRI.getUniqueVRegDef(Reg: Op1.getReg());
2432 if (Op2.isReg() && Op2.getReg().isVirtual())
2433 MI2 = MRI.getUniqueVRegDef(Reg: Op2.getReg());
2434
2435 // And at least one operand must be defined in MBB.
2436 return MI1 && MI2 && (MI1->getParent() == MBB || MI2->getParent() == MBB);
2437}
2438
2439void RISCVInstrInfo::getReassociateOperandIndices(
2440 const MachineInstr &Root, unsigned Pattern,
2441 std::array<unsigned, 5> &OperandIndices) const {
2442 TargetInstrInfo::getReassociateOperandIndices(Root, Pattern, OperandIndices);
2443 if (RISCV::getRVVMCOpcode(RVVPseudoOpcode: Root.getOpcode())) {
2444 // Skip the passthrough operand, so increment all indices by one.
2445 for (unsigned I = 0; I < 5; ++I)
2446 ++OperandIndices[I];
2447 }
2448}
2449
2450bool RISCVInstrInfo::hasReassociableSibling(const MachineInstr &Inst,
2451 bool &Commuted) const {
2452 if (isVectorAssociativeAndCommutative(Inst) ||
2453 isVectorAssociativeAndCommutative(Inst, /*Invert=*/true))
2454 return hasReassociableVectorSibling(Inst, Commuted);
2455
2456 if (!TargetInstrInfo::hasReassociableSibling(Inst, Commuted))
2457 return false;
2458
2459 const MachineRegisterInfo &MRI = Inst.getMF()->getRegInfo();
2460 unsigned OperandIdx = Commuted ? 2 : 1;
2461 const MachineInstr &Sibling =
2462 *MRI.getVRegDef(Reg: Inst.getOperand(i: OperandIdx).getReg());
2463
2464 int16_t InstFrmOpIdx =
2465 RISCV::getNamedOperandIdx(Opcode: Inst.getOpcode(), Name: RISCV::OpName::frm);
2466 int16_t SiblingFrmOpIdx =
2467 RISCV::getNamedOperandIdx(Opcode: Sibling.getOpcode(), Name: RISCV::OpName::frm);
2468
2469 return (InstFrmOpIdx < 0 && SiblingFrmOpIdx < 0) ||
2470 RISCV::hasEqualFRM(MI1: Inst, MI2: Sibling);
2471}
2472
2473bool RISCVInstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst,
2474 bool Invert) const {
2475 if (isVectorAssociativeAndCommutative(Inst, Invert))
2476 return true;
2477
2478 unsigned Opc = Inst.getOpcode();
2479 if (Invert) {
2480 auto InverseOpcode = getInverseOpcode(Opcode: Opc);
2481 if (!InverseOpcode)
2482 return false;
2483 Opc = *InverseOpcode;
2484 }
2485
2486 if (isFADD(Opc) || isFMUL(Opc))
2487 return Inst.getFlag(Flag: MachineInstr::MIFlag::FmReassoc) &&
2488 Inst.getFlag(Flag: MachineInstr::MIFlag::FmNsz);
2489
2490 switch (Opc) {
2491 default:
2492 return false;
2493 case RISCV::ADD:
2494 case RISCV::ADDW:
2495 case RISCV::AND:
2496 case RISCV::OR:
2497 case RISCV::XOR:
2498 // From RISC-V ISA spec, if both the high and low bits of the same product
2499 // are required, then the recommended code sequence is:
2500 //
2501 // MULH[[S]U] rdh, rs1, rs2
2502 // MUL rdl, rs1, rs2
2503 // (source register specifiers must be in same order and rdh cannot be the
2504 // same as rs1 or rs2)
2505 //
2506 // Microarchitectures can then fuse these into a single multiply operation
2507 // instead of performing two separate multiplies.
2508 // MachineCombiner may reassociate MUL operands and lose the fusion
2509 // opportunity.
2510 case RISCV::MUL:
2511 case RISCV::MULW:
2512 case RISCV::MIN:
2513 case RISCV::MINU:
2514 case RISCV::MAX:
2515 case RISCV::MAXU:
2516 case RISCV::FMIN_H:
2517 case RISCV::FMIN_S:
2518 case RISCV::FMIN_D:
2519 case RISCV::FMAX_H:
2520 case RISCV::FMAX_S:
2521 case RISCV::FMAX_D:
2522 return true;
2523 }
2524
2525 return false;
2526}
2527
2528std::optional<unsigned>
2529RISCVInstrInfo::getInverseOpcode(unsigned Opcode) const {
2530#define RVV_OPC_LMUL_CASE(OPC, INV) \
2531 case RISCV::OPC##_M1: \
2532 return RISCV::INV##_M1; \
2533 case RISCV::OPC##_M2: \
2534 return RISCV::INV##_M2; \
2535 case RISCV::OPC##_M4: \
2536 return RISCV::INV##_M4; \
2537 case RISCV::OPC##_M8: \
2538 return RISCV::INV##_M8; \
2539 case RISCV::OPC##_MF2: \
2540 return RISCV::INV##_MF2; \
2541 case RISCV::OPC##_MF4: \
2542 return RISCV::INV##_MF4; \
2543 case RISCV::OPC##_MF8: \
2544 return RISCV::INV##_MF8
2545
2546#define RVV_OPC_LMUL_MASK_CASE(OPC, INV) \
2547 case RISCV::OPC##_M1_MASK: \
2548 return RISCV::INV##_M1_MASK; \
2549 case RISCV::OPC##_M2_MASK: \
2550 return RISCV::INV##_M2_MASK; \
2551 case RISCV::OPC##_M4_MASK: \
2552 return RISCV::INV##_M4_MASK; \
2553 case RISCV::OPC##_M8_MASK: \
2554 return RISCV::INV##_M8_MASK; \
2555 case RISCV::OPC##_MF2_MASK: \
2556 return RISCV::INV##_MF2_MASK; \
2557 case RISCV::OPC##_MF4_MASK: \
2558 return RISCV::INV##_MF4_MASK; \
2559 case RISCV::OPC##_MF8_MASK: \
2560 return RISCV::INV##_MF8_MASK
2561
2562 switch (Opcode) {
2563 default:
2564 return std::nullopt;
2565 case RISCV::FADD_H:
2566 return RISCV::FSUB_H;
2567 case RISCV::FADD_S:
2568 return RISCV::FSUB_S;
2569 case RISCV::FADD_D:
2570 return RISCV::FSUB_D;
2571 case RISCV::FSUB_H:
2572 return RISCV::FADD_H;
2573 case RISCV::FSUB_S:
2574 return RISCV::FADD_S;
2575 case RISCV::FSUB_D:
2576 return RISCV::FADD_D;
2577 case RISCV::ADD:
2578 return RISCV::SUB;
2579 case RISCV::SUB:
2580 return RISCV::ADD;
2581 case RISCV::ADDW:
2582 return RISCV::SUBW;
2583 case RISCV::SUBW:
2584 return RISCV::ADDW;
2585 // clang-format off
2586 RVV_OPC_LMUL_CASE(PseudoVADD_VV, PseudoVSUB_VV);
2587 RVV_OPC_LMUL_MASK_CASE(PseudoVADD_VV, PseudoVSUB_VV);
2588 RVV_OPC_LMUL_CASE(PseudoVSUB_VV, PseudoVADD_VV);
2589 RVV_OPC_LMUL_MASK_CASE(PseudoVSUB_VV, PseudoVADD_VV);
2590 // clang-format on
2591 }
2592
2593#undef RVV_OPC_LMUL_MASK_CASE
2594#undef RVV_OPC_LMUL_CASE
2595}
2596
2597static bool canCombineFPFusedMultiply(const MachineInstr &Root,
2598 const MachineOperand &MO,
2599 bool DoRegPressureReduce) {
2600 if (!MO.isReg() || !MO.getReg().isVirtual())
2601 return false;
2602 const MachineRegisterInfo &MRI = Root.getMF()->getRegInfo();
2603 MachineInstr *MI = MRI.getVRegDef(Reg: MO.getReg());
2604 if (!MI || !isFMUL(Opc: MI->getOpcode()))
2605 return false;
2606
2607 if (!Root.getFlag(Flag: MachineInstr::MIFlag::FmContract) ||
2608 !MI->getFlag(Flag: MachineInstr::MIFlag::FmContract))
2609 return false;
2610
2611 // Try combining even if fmul has more than one use as it eliminates
2612 // dependency between fadd(fsub) and fmul. However, it can extend liveranges
2613 // for fmul operands, so reject the transformation in register pressure
2614 // reduction mode.
2615 if (DoRegPressureReduce && !MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()))
2616 return false;
2617
2618 // Do not combine instructions from different basic blocks.
2619 if (Root.getParent() != MI->getParent())
2620 return false;
2621 return RISCV::hasEqualFRM(MI1: Root, MI2: *MI);
2622}
2623
2624static bool getFPFusedMultiplyPatterns(MachineInstr &Root,
2625 SmallVectorImpl<unsigned> &Patterns,
2626 bool DoRegPressureReduce) {
2627 unsigned Opc = Root.getOpcode();
2628 bool IsFAdd = isFADD(Opc);
2629 if (!IsFAdd && !isFSUB(Opc))
2630 return false;
2631 bool Added = false;
2632 if (canCombineFPFusedMultiply(Root, MO: Root.getOperand(i: 1),
2633 DoRegPressureReduce)) {
2634 Patterns.push_back(Elt: IsFAdd ? RISCVMachineCombinerPattern::FMADD_AX
2635 : RISCVMachineCombinerPattern::FMSUB);
2636 Added = true;
2637 }
2638 if (canCombineFPFusedMultiply(Root, MO: Root.getOperand(i: 2),
2639 DoRegPressureReduce)) {
2640 Patterns.push_back(Elt: IsFAdd ? RISCVMachineCombinerPattern::FMADD_XA
2641 : RISCVMachineCombinerPattern::FNMSUB);
2642 Added = true;
2643 }
2644 return Added;
2645}
2646
2647static bool getFPPatterns(MachineInstr &Root,
2648 SmallVectorImpl<unsigned> &Patterns,
2649 bool DoRegPressureReduce) {
2650 return getFPFusedMultiplyPatterns(Root, Patterns, DoRegPressureReduce);
2651}
2652
2653/// Utility routine that checks if \param MO is defined by an
2654/// \param CombineOpc instruction in the basic block \param MBB
2655static const MachineInstr *canCombine(const MachineBasicBlock &MBB,
2656 const MachineOperand &MO,
2657 unsigned CombineOpc) {
2658 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2659 const MachineInstr *MI = nullptr;
2660
2661 if (MO.isReg() && MO.getReg().isVirtual())
2662 MI = MRI.getUniqueVRegDef(Reg: MO.getReg());
2663 // And it needs to be in the trace (otherwise, it won't have a depth).
2664 if (!MI || MI->getParent() != &MBB || MI->getOpcode() != CombineOpc)
2665 return nullptr;
2666 // Must only used by the user we combine with.
2667 if (!MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()))
2668 return nullptr;
2669
2670 return MI;
2671}
2672
2673/// Utility routine that checks if \param MO is defined by a SLLI in \param
2674/// MBB that can be combined by splitting across 2 SHXADD instructions. The
2675/// first SHXADD shift amount is given by \param OuterShiftAmt.
2676static bool canCombineShiftIntoShXAdd(const MachineBasicBlock &MBB,
2677 const MachineOperand &MO,
2678 unsigned OuterShiftAmt) {
2679 const MachineInstr *ShiftMI = canCombine(MBB, MO, CombineOpc: RISCV::SLLI);
2680 if (!ShiftMI)
2681 return false;
2682
2683 unsigned InnerShiftAmt = ShiftMI->getOperand(i: 2).getImm();
2684 if (InnerShiftAmt < OuterShiftAmt || (InnerShiftAmt - OuterShiftAmt) > 3)
2685 return false;
2686
2687 return true;
2688}
2689
2690// Returns the shift amount from a SHXADD instruction. Returns 0 if the
2691// instruction is not a SHXADD.
2692static unsigned getSHXADDShiftAmount(unsigned Opc) {
2693 switch (Opc) {
2694 default:
2695 return 0;
2696 case RISCV::SH1ADD:
2697 return 1;
2698 case RISCV::SH2ADD:
2699 return 2;
2700 case RISCV::SH3ADD:
2701 return 3;
2702 }
2703}
2704
2705// Returns the shift amount from a SHXADD.UW instruction. Returns 0 if the
2706// instruction is not a SHXADD.UW.
2707static unsigned getSHXADDUWShiftAmount(unsigned Opc) {
2708 switch (Opc) {
2709 default:
2710 return 0;
2711 case RISCV::SH1ADD_UW:
2712 return 1;
2713 case RISCV::SH2ADD_UW:
2714 return 2;
2715 case RISCV::SH3ADD_UW:
2716 return 3;
2717 }
2718}
2719
2720// Look for opportunities to combine (sh3add Z, (add X, (slli Y, 5))) into
2721// (sh3add (sh2add Y, Z), X).
2722static bool getSHXADDPatterns(const MachineInstr &Root,
2723 SmallVectorImpl<unsigned> &Patterns) {
2724 unsigned ShiftAmt = getSHXADDShiftAmount(Opc: Root.getOpcode());
2725 if (!ShiftAmt)
2726 return false;
2727
2728 const MachineBasicBlock &MBB = *Root.getParent();
2729
2730 const MachineInstr *AddMI = canCombine(MBB, MO: Root.getOperand(i: 2), CombineOpc: RISCV::ADD);
2731 if (!AddMI)
2732 return false;
2733
2734 bool Found = false;
2735 if (canCombineShiftIntoShXAdd(MBB, MO: AddMI->getOperand(i: 1), OuterShiftAmt: ShiftAmt)) {
2736 Patterns.push_back(Elt: RISCVMachineCombinerPattern::SHXADD_ADD_SLLI_OP1);
2737 Found = true;
2738 }
2739 if (canCombineShiftIntoShXAdd(MBB, MO: AddMI->getOperand(i: 2), OuterShiftAmt: ShiftAmt)) {
2740 Patterns.push_back(Elt: RISCVMachineCombinerPattern::SHXADD_ADD_SLLI_OP2);
2741 Found = true;
2742 }
2743
2744 return Found;
2745}
2746
2747CombinerObjective RISCVInstrInfo::getCombinerObjective(unsigned Pattern) const {
2748 switch (Pattern) {
2749 case RISCVMachineCombinerPattern::FMADD_AX:
2750 case RISCVMachineCombinerPattern::FMADD_XA:
2751 case RISCVMachineCombinerPattern::FMSUB:
2752 case RISCVMachineCombinerPattern::FNMSUB:
2753 return CombinerObjective::MustReduceDepth;
2754 default:
2755 return TargetInstrInfo::getCombinerObjective(Pattern);
2756 }
2757}
2758
2759bool RISCVInstrInfo::getMachineCombinerPatterns(
2760 MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns,
2761 bool DoRegPressureReduce) const {
2762
2763 if (getFPPatterns(Root, Patterns, DoRegPressureReduce))
2764 return true;
2765
2766 if (getSHXADDPatterns(Root, Patterns))
2767 return true;
2768
2769 return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
2770 DoRegPressureReduce);
2771}
2772
2773static unsigned getFPFusedMultiplyOpcode(unsigned RootOpc, unsigned Pattern) {
2774 switch (RootOpc) {
2775 default:
2776 llvm_unreachable("Unexpected opcode");
2777 case RISCV::FADD_H:
2778 return RISCV::FMADD_H;
2779 case RISCV::FADD_S:
2780 return RISCV::FMADD_S;
2781 case RISCV::FADD_D:
2782 return RISCV::FMADD_D;
2783 case RISCV::FSUB_H:
2784 return Pattern == RISCVMachineCombinerPattern::FMSUB ? RISCV::FMSUB_H
2785 : RISCV::FNMSUB_H;
2786 case RISCV::FSUB_S:
2787 return Pattern == RISCVMachineCombinerPattern::FMSUB ? RISCV::FMSUB_S
2788 : RISCV::FNMSUB_S;
2789 case RISCV::FSUB_D:
2790 return Pattern == RISCVMachineCombinerPattern::FMSUB ? RISCV::FMSUB_D
2791 : RISCV::FNMSUB_D;
2792 }
2793}
2794
2795static unsigned getAddendOperandIdx(unsigned Pattern) {
2796 switch (Pattern) {
2797 default:
2798 llvm_unreachable("Unexpected pattern");
2799 case RISCVMachineCombinerPattern::FMADD_AX:
2800 case RISCVMachineCombinerPattern::FMSUB:
2801 return 2;
2802 case RISCVMachineCombinerPattern::FMADD_XA:
2803 case RISCVMachineCombinerPattern::FNMSUB:
2804 return 1;
2805 }
2806}
2807
2808static void combineFPFusedMultiply(MachineInstr &Root, MachineInstr &Prev,
2809 unsigned Pattern,
2810 SmallVectorImpl<MachineInstr *> &InsInstrs,
2811 SmallVectorImpl<MachineInstr *> &DelInstrs) {
2812 MachineFunction *MF = Root.getMF();
2813 MachineRegisterInfo &MRI = MF->getRegInfo();
2814 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
2815
2816 MachineOperand &Mul1 = Prev.getOperand(i: 1);
2817 MachineOperand &Mul2 = Prev.getOperand(i: 2);
2818 MachineOperand &Dst = Root.getOperand(i: 0);
2819 MachineOperand &Addend = Root.getOperand(i: getAddendOperandIdx(Pattern));
2820
2821 Register DstReg = Dst.getReg();
2822 unsigned FusedOpc = getFPFusedMultiplyOpcode(RootOpc: Root.getOpcode(), Pattern);
2823 uint32_t IntersectedFlags = Root.getFlags() & Prev.getFlags();
2824 DebugLoc MergedLoc =
2825 DILocation::getMergedLocation(LocA: Root.getDebugLoc(), LocB: Prev.getDebugLoc());
2826
2827 bool Mul1IsKill = Mul1.isKill();
2828 bool Mul2IsKill = Mul2.isKill();
2829 bool AddendIsKill = Addend.isKill();
2830
2831 // We need to clear kill flags since we may be extending the live range past
2832 // a kill. If the mul had kill flags, we can preserve those since we know
2833 // where the previous range stopped.
2834 MRI.clearKillFlags(Reg: Mul1.getReg());
2835 MRI.clearKillFlags(Reg: Mul2.getReg());
2836
2837 MachineInstrBuilder MIB =
2838 BuildMI(MF&: *MF, MIMD: MergedLoc, MCID: TII->get(Opcode: FusedOpc), DestReg: DstReg)
2839 .addReg(RegNo: Mul1.getReg(), Flags: getKillRegState(B: Mul1IsKill))
2840 .addReg(RegNo: Mul2.getReg(), Flags: getKillRegState(B: Mul2IsKill))
2841 .addReg(RegNo: Addend.getReg(), Flags: getKillRegState(B: AddendIsKill))
2842 .setMIFlags(IntersectedFlags);
2843
2844 InsInstrs.push_back(Elt: MIB);
2845 if (MRI.hasOneNonDBGUse(RegNo: Prev.getOperand(i: 0).getReg()))
2846 DelInstrs.push_back(Elt: &Prev);
2847 DelInstrs.push_back(Elt: &Root);
2848}
2849
2850// Combine patterns like (sh3add Z, (add X, (slli Y, 5))) to
2851// (sh3add (sh2add Y, Z), X) if the shift amount can be split across two
2852// shXadd instructions. The outer shXadd keeps its original opcode.
2853static void
2854genShXAddAddShift(MachineInstr &Root, unsigned AddOpIdx,
2855 SmallVectorImpl<MachineInstr *> &InsInstrs,
2856 SmallVectorImpl<MachineInstr *> &DelInstrs,
2857 DenseMap<Register, unsigned> &InstrIdxForVirtReg) {
2858 MachineFunction *MF = Root.getMF();
2859 MachineRegisterInfo &MRI = MF->getRegInfo();
2860 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
2861
2862 unsigned OuterShiftAmt = getSHXADDShiftAmount(Opc: Root.getOpcode());
2863 assert(OuterShiftAmt != 0 && "Unexpected opcode");
2864
2865 MachineInstr *AddMI = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: 2).getReg());
2866 MachineInstr *ShiftMI =
2867 MRI.getUniqueVRegDef(Reg: AddMI->getOperand(i: AddOpIdx).getReg());
2868
2869 unsigned InnerShiftAmt = ShiftMI->getOperand(i: 2).getImm();
2870 assert(InnerShiftAmt >= OuterShiftAmt && "Unexpected shift amount");
2871
2872 unsigned InnerOpc;
2873 switch (InnerShiftAmt - OuterShiftAmt) {
2874 default:
2875 llvm_unreachable("Unexpected shift amount");
2876 case 0:
2877 InnerOpc = RISCV::ADD;
2878 break;
2879 case 1:
2880 InnerOpc = RISCV::SH1ADD;
2881 break;
2882 case 2:
2883 InnerOpc = RISCV::SH2ADD;
2884 break;
2885 case 3:
2886 InnerOpc = RISCV::SH3ADD;
2887 break;
2888 }
2889
2890 const MachineOperand &X = AddMI->getOperand(i: 3 - AddOpIdx);
2891 const MachineOperand &Y = ShiftMI->getOperand(i: 1);
2892 const MachineOperand &Z = Root.getOperand(i: 1);
2893
2894 Register NewVR = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
2895
2896 auto MIB1 = BuildMI(MF&: *MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: InnerOpc), DestReg: NewVR)
2897 .addReg(RegNo: Y.getReg(), Flags: getKillRegState(B: Y.isKill()))
2898 .addReg(RegNo: Z.getReg(), Flags: getKillRegState(B: Z.isKill()));
2899 auto MIB2 = BuildMI(MF&: *MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: Root.getOpcode()),
2900 DestReg: Root.getOperand(i: 0).getReg())
2901 .addReg(RegNo: NewVR, Flags: RegState::Kill)
2902 .addReg(RegNo: X.getReg(), Flags: getKillRegState(B: X.isKill()));
2903
2904 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
2905 InsInstrs.push_back(Elt: MIB1);
2906 InsInstrs.push_back(Elt: MIB2);
2907 DelInstrs.push_back(Elt: ShiftMI);
2908 DelInstrs.push_back(Elt: AddMI);
2909 DelInstrs.push_back(Elt: &Root);
2910}
2911
2912void RISCVInstrInfo::genAlternativeCodeSequence(
2913 MachineInstr &Root, unsigned Pattern,
2914 SmallVectorImpl<MachineInstr *> &InsInstrs,
2915 SmallVectorImpl<MachineInstr *> &DelInstrs,
2916 DenseMap<Register, unsigned> &InstrIdxForVirtReg) const {
2917 MachineRegisterInfo &MRI = Root.getMF()->getRegInfo();
2918 switch (Pattern) {
2919 default:
2920 TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs,
2921 DelInstrs, InstIdxForVirtReg&: InstrIdxForVirtReg);
2922 return;
2923 case RISCVMachineCombinerPattern::FMADD_AX:
2924 case RISCVMachineCombinerPattern::FMSUB: {
2925 MachineInstr &Prev = *MRI.getVRegDef(Reg: Root.getOperand(i: 1).getReg());
2926 combineFPFusedMultiply(Root, Prev, Pattern, InsInstrs, DelInstrs);
2927 return;
2928 }
2929 case RISCVMachineCombinerPattern::FMADD_XA:
2930 case RISCVMachineCombinerPattern::FNMSUB: {
2931 MachineInstr &Prev = *MRI.getVRegDef(Reg: Root.getOperand(i: 2).getReg());
2932 combineFPFusedMultiply(Root, Prev, Pattern, InsInstrs, DelInstrs);
2933 return;
2934 }
2935 case RISCVMachineCombinerPattern::SHXADD_ADD_SLLI_OP1:
2936 genShXAddAddShift(Root, AddOpIdx: 1, InsInstrs, DelInstrs, InstrIdxForVirtReg);
2937 return;
2938 case RISCVMachineCombinerPattern::SHXADD_ADD_SLLI_OP2:
2939 genShXAddAddShift(Root, AddOpIdx: 2, InsInstrs, DelInstrs, InstrIdxForVirtReg);
2940 return;
2941 }
2942}
2943
2944bool RISCVInstrInfo::verifyInstruction(const MachineInstr &MI,
2945 StringRef &ErrInfo) const {
2946 MCInstrDesc const &Desc = MI.getDesc();
2947
2948 for (const auto &[Index, Operand] : enumerate(First: Desc.operands())) {
2949 const MachineOperand &MO = MI.getOperand(i: Index);
2950 unsigned OpType = Operand.OperandType;
2951 switch (OpType) {
2952 default:
2953 if (OpType >= RISCVOp::OPERAND_FIRST_RISCV_IMM &&
2954 OpType <= RISCVOp::OPERAND_LAST_RISCV_IMM) {
2955 if (!MO.isImm()) {
2956 ErrInfo = "Expected an immediate operand.";
2957 return false;
2958 }
2959 int64_t Imm = MO.getImm();
2960 bool Ok;
2961 switch (OpType) {
2962 default:
2963 llvm_unreachable("Unexpected operand type");
2964
2965#define CASE_OPERAND_UIMM(NUM) \
2966 case RISCVOp::OPERAND_UIMM##NUM: \
2967 Ok = isUInt<NUM>(Imm); \
2968 break;
2969#define CASE_OPERAND_UIMM_LSB_ZEROS(BITS, SUFFIX) \
2970 case RISCVOp::OPERAND_UIMM##BITS##_LSB##SUFFIX: { \
2971 constexpr size_t NumZeros = sizeof(#SUFFIX) - 1; \
2972 Ok = isShiftedUInt<BITS - NumZeros, NumZeros>(Imm); \
2973 break; \
2974 }
2975#define CASE_OPERAND_SIMM(NUM) \
2976 case RISCVOp::OPERAND_SIMM##NUM: \
2977 Ok = isInt<NUM>(Imm); \
2978 break;
2979 // clang-format off
2980 CASE_OPERAND_UIMM(1)
2981 CASE_OPERAND_UIMM(2)
2982 CASE_OPERAND_UIMM(3)
2983 CASE_OPERAND_UIMM(4)
2984 CASE_OPERAND_UIMM(5)
2985 CASE_OPERAND_UIMM(6)
2986 CASE_OPERAND_UIMM(7)
2987 CASE_OPERAND_UIMM(8)
2988 CASE_OPERAND_UIMM(9)
2989 CASE_OPERAND_UIMM(10)
2990 CASE_OPERAND_UIMM(12)
2991 CASE_OPERAND_UIMM(16)
2992 CASE_OPERAND_UIMM(32)
2993 CASE_OPERAND_UIMM(48)
2994 CASE_OPERAND_UIMM(64)
2995 CASE_OPERAND_UIMM_LSB_ZEROS(2, 0)
2996 CASE_OPERAND_UIMM_LSB_ZEROS(5, 0)
2997 CASE_OPERAND_UIMM_LSB_ZEROS(6, 0)
2998 CASE_OPERAND_UIMM_LSB_ZEROS(7, 00)
2999 CASE_OPERAND_UIMM_LSB_ZEROS(7, 000)
3000 CASE_OPERAND_UIMM_LSB_ZEROS(8, 00)
3001 CASE_OPERAND_UIMM_LSB_ZEROS(8, 000)
3002 CASE_OPERAND_UIMM_LSB_ZEROS(9, 000)
3003 // clang-format on
3004 case RISCVOp::OPERAND_UIMM5_NONZERO:
3005 Ok = isUInt<5>(x: Imm) && (Imm != 0);
3006 break;
3007 case RISCVOp::OPERAND_UIMM5_GT3:
3008 Ok = isUInt<5>(x: Imm) && (Imm > 3);
3009 break;
3010 case RISCVOp::OPERAND_UIMM5_PLUS1:
3011 Ok = Imm >= 1 && Imm <= 32;
3012 break;
3013 case RISCVOp::OPERAND_UIMM8_GE32:
3014 Ok = isUInt<8>(x: Imm) && Imm >= 32;
3015 break;
3016 case RISCVOp::OPERAND_SIMM8_UNSIGNED:
3017 Ok = isInt<8>(x: Imm);
3018 break;
3019 case RISCVOp::OPERAND_SIMM10_LSB0000_NONZERO:
3020 Ok = isShiftedInt<6, 4>(x: Imm) && (Imm != 0);
3021 break;
3022 case RISCVOp::OPERAND_UIMM10_LSB00_NONZERO:
3023 Ok = isShiftedUInt<8, 2>(x: Imm) && (Imm != 0);
3024 break;
3025 case RISCVOp::OPERAND_UIMM16_NONZERO:
3026 Ok = isUInt<16>(x: Imm) && (Imm != 0);
3027 break;
3028 case RISCVOp::OPERAND_THREE:
3029 Ok = Imm == 3;
3030 break;
3031 case RISCVOp::OPERAND_FOUR:
3032 Ok = Imm == 4;
3033 break;
3034 case RISCVOp::OPERAND_IMM5_ZIBI:
3035 Ok = (isUInt<5>(x: Imm) && Imm != 0) || Imm == -1;
3036 break;
3037 // clang-format off
3038 CASE_OPERAND_SIMM(5)
3039 CASE_OPERAND_SIMM(6)
3040 CASE_OPERAND_SIMM(10)
3041 CASE_OPERAND_SIMM(11)
3042 CASE_OPERAND_SIMM(26)
3043 // clang-format on
3044 case RISCVOp::OPERAND_SIMM5_PLUS1:
3045 Ok = Imm >= -15 && Imm <= 16;
3046 break;
3047 case RISCVOp::OPERAND_SIMM5_NONZERO:
3048 Ok = isInt<5>(x: Imm) && (Imm != 0);
3049 break;
3050 case RISCVOp::OPERAND_SIMM6_NONZERO:
3051 Ok = Imm != 0 && isInt<6>(x: Imm);
3052 break;
3053 case RISCVOp::OPERAND_VTYPEI10:
3054 Ok = isUInt<10>(x: Imm);
3055 break;
3056 case RISCVOp::OPERAND_VTYPEI11:
3057 Ok = isUInt<11>(x: Imm);
3058 break;
3059 case RISCVOp::OPERAND_SIMM12_LSB00000:
3060 Ok = isShiftedInt<7, 5>(x: Imm);
3061 break;
3062 case RISCVOp::OPERAND_SIMM16_NONZERO:
3063 Ok = isInt<16>(x: Imm) && (Imm != 0);
3064 break;
3065 case RISCVOp::OPERAND_SIMM20_LI:
3066 Ok = isInt<20>(x: Imm);
3067 break;
3068 case RISCVOp::OPERAND_UIMMLOG2XLEN:
3069 Ok = STI.is64Bit() ? isUInt<6>(x: Imm) : isUInt<5>(x: Imm);
3070 break;
3071 case RISCVOp::OPERAND_UIMMLOG2XLEN_NONZERO:
3072 Ok = STI.is64Bit() ? isUInt<6>(x: Imm) : isUInt<5>(x: Imm);
3073 Ok = Ok && Imm != 0;
3074 break;
3075 case RISCVOp::OPERAND_CLUI_IMM:
3076 Ok = (isUInt<5>(x: Imm) && Imm != 0) || (Imm >= 0xfffe0 && Imm <= 0xfffff);
3077 break;
3078 case RISCVOp::OPERAND_RVKRNUM:
3079 Ok = Imm >= 0 && Imm <= 10;
3080 break;
3081 case RISCVOp::OPERAND_RVKRNUM_0_7:
3082 Ok = Imm >= 0 && Imm <= 7;
3083 break;
3084 case RISCVOp::OPERAND_RVKRNUM_1_10:
3085 Ok = Imm >= 1 && Imm <= 10;
3086 break;
3087 case RISCVOp::OPERAND_RVKRNUM_2_14:
3088 Ok = Imm >= 2 && Imm <= 14;
3089 break;
3090 case RISCVOp::OPERAND_RLIST:
3091 Ok = Imm >= RISCVZC::RA && Imm <= RISCVZC::RA_S0_S11;
3092 break;
3093 case RISCVOp::OPERAND_RLIST_S0:
3094 Ok = Imm >= RISCVZC::RA_S0 && Imm <= RISCVZC::RA_S0_S11;
3095 break;
3096 case RISCVOp::OPERAND_STACKADJ:
3097 Ok = Imm >= 0 && Imm <= 48 && Imm % 16 == 0;
3098 break;
3099 case RISCVOp::OPERAND_FRMARG:
3100 Ok = RISCVFPRndMode::isValidRoundingMode(Mode: Imm);
3101 break;
3102 case RISCVOp::OPERAND_RTZARG:
3103 Ok = Imm == RISCVFPRndMode::RTZ;
3104 break;
3105 case RISCVOp::OPERAND_COND_CODE:
3106 Ok = Imm >= 0 && Imm < RISCVCC::COND_INVALID;
3107 break;
3108 case RISCVOp::OPERAND_ATOMIC_ORDERING:
3109 Ok = isValidAtomicOrdering(I: Imm);
3110 break;
3111 case RISCVOp::OPERAND_VEC_POLICY:
3112 Ok = (Imm & (RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC)) ==
3113 Imm;
3114 break;
3115 case RISCVOp::OPERAND_SEW:
3116 Ok = (isUInt<5>(x: Imm) && RISCVVType::isValidSEW(SEW: 1 << Imm));
3117 break;
3118 case RISCVOp::OPERAND_SEW_MASK:
3119 Ok = Imm == 0;
3120 break;
3121 case RISCVOp::OPERAND_VEC_RM:
3122 assert(RISCVII::hasRoundModeOp(Desc.TSFlags));
3123 if (RISCVII::usesVXRM(TSFlags: Desc.TSFlags))
3124 Ok = isUInt<2>(x: Imm);
3125 else
3126 Ok = RISCVFPRndMode::isValidRoundingMode(Mode: Imm);
3127 break;
3128 case RISCVOp::OPERAND_XSFMM_VTYPE:
3129 Ok = RISCVVType::isValidXSfmmVType(VTypeI: Imm);
3130 break;
3131 case RISCVOp::OPERAND_XSFMM_TWIDEN:
3132 Ok = Imm == 1 || Imm == 2 || Imm == 4;
3133 break;
3134 }
3135 if (!Ok) {
3136 ErrInfo = "Invalid immediate";
3137 return false;
3138 }
3139 }
3140 break;
3141 case RISCVOp::OPERAND_SIMM12_LO:
3142 // TODO: We could be stricter about what non-register operands are
3143 // allowed.
3144 if (MO.isReg()) {
3145 ErrInfo = "Expected a non-register operand.";
3146 return false;
3147 }
3148 if (MO.isImm() && !isInt<12>(x: MO.getImm())) {
3149 ErrInfo = "Invalid immediate";
3150 return false;
3151 }
3152 break;
3153 case RISCVOp::OPERAND_UIMM20_LUI:
3154 case RISCVOp::OPERAND_UIMM20_AUIPC:
3155 // TODO: We could be stricter about what non-register operands are
3156 // allowed.
3157 if (MO.isReg()) {
3158 ErrInfo = "Expected a non-register operand.";
3159 return false;
3160 }
3161 if (MO.isImm() && !isUInt<20>(x: MO.getImm())) {
3162 ErrInfo = "Invalid immediate";
3163 return false;
3164 }
3165 break;
3166 case RISCVOp::OPERAND_BARE_SIMM32:
3167 // TODO: We could be stricter about what non-register operands are
3168 // allowed.
3169 if (MO.isReg()) {
3170 ErrInfo = "Expected a non-register operand.";
3171 return false;
3172 }
3173 if (MO.isImm() && !isInt<32>(x: MO.getImm())) {
3174 ErrInfo = "Invalid immediate";
3175 return false;
3176 }
3177 break;
3178 case RISCVOp::OPERAND_AVL:
3179 if (MO.isImm()) {
3180 int64_t Imm = MO.getImm();
3181 // VLMAX is represented as -1.
3182 if (!isUInt<5>(x: Imm) && Imm != -1) {
3183 ErrInfo = "Invalid immediate";
3184 return false;
3185 }
3186 } else if (!MO.isReg()) {
3187 ErrInfo = "Expected a register or immediate operand.";
3188 return false;
3189 }
3190 break;
3191 }
3192 }
3193
3194 const uint64_t TSFlags = Desc.TSFlags;
3195 if (RISCVII::hasVLOp(TSFlags)) {
3196 const MachineOperand &Op = MI.getOperand(i: RISCVII::getVLOpNum(Desc));
3197 if (!Op.isImm() && !Op.isReg()) {
3198 ErrInfo = "Invalid operand type for VL operand";
3199 return false;
3200 }
3201 if (Op.isReg() && Op.getReg().isValid()) {
3202 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
3203 auto *RC = MRI.getRegClass(Reg: Op.getReg());
3204 if (!RISCV::GPRNoX0RegClass.hasSubClassEq(RC)) {
3205 ErrInfo = "Invalid register class for VL operand";
3206 return false;
3207 }
3208 }
3209 if (!RISCVII::hasSEWOp(TSFlags)) {
3210 ErrInfo = "VL operand w/o SEW operand?";
3211 return false;
3212 }
3213 }
3214 if (RISCVII::hasSEWOp(TSFlags)) {
3215 unsigned OpIdx = RISCVII::getSEWOpNum(Desc);
3216 if (!MI.getOperand(i: OpIdx).isImm()) {
3217 ErrInfo = "SEW value expected to be an immediate";
3218 return false;
3219 }
3220 uint64_t Log2SEW = MI.getOperand(i: OpIdx).getImm();
3221 if (Log2SEW > 31) {
3222 ErrInfo = "Unexpected SEW value";
3223 return false;
3224 }
3225 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
3226 if (!RISCVVType::isValidSEW(SEW)) {
3227 ErrInfo = "Unexpected SEW value";
3228 return false;
3229 }
3230 }
3231 if (RISCVII::hasVecPolicyOp(TSFlags)) {
3232 unsigned OpIdx = RISCVII::getVecPolicyOpNum(Desc);
3233 if (!MI.getOperand(i: OpIdx).isImm()) {
3234 ErrInfo = "Policy operand expected to be an immediate";
3235 return false;
3236 }
3237 uint64_t Policy = MI.getOperand(i: OpIdx).getImm();
3238 if (Policy > (RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC)) {
3239 ErrInfo = "Invalid Policy Value";
3240 return false;
3241 }
3242 if (!RISCVII::hasVLOp(TSFlags)) {
3243 ErrInfo = "policy operand w/o VL operand?";
3244 return false;
3245 }
3246
3247 // VecPolicy operands can only exist on instructions with passthru/merge
3248 // arguments. Note that not all arguments with passthru have vec policy
3249 // operands- some instructions have implicit policies.
3250 unsigned UseOpIdx;
3251 if (!MI.isRegTiedToUseOperand(DefOpIdx: 0, UseOpIdx: &UseOpIdx)) {
3252 ErrInfo = "policy operand w/o tied operand?";
3253 return false;
3254 }
3255 }
3256
3257 if (int Idx = RISCVII::getFRMOpNum(Desc);
3258 Idx >= 0 && MI.getOperand(i: Idx).getImm() == RISCVFPRndMode::DYN &&
3259 !MI.readsRegister(Reg: RISCV::FRM, /*TRI=*/nullptr)) {
3260 ErrInfo = "dynamic rounding mode should read FRM";
3261 return false;
3262 }
3263
3264 return true;
3265}
3266
3267bool RISCVInstrInfo::canFoldIntoAddrMode(const MachineInstr &MemI, Register Reg,
3268 const MachineInstr &AddrI,
3269 ExtAddrMode &AM) const {
3270 switch (MemI.getOpcode()) {
3271 default:
3272 return false;
3273 case RISCV::LB:
3274 case RISCV::LBU:
3275 case RISCV::LH:
3276 case RISCV::LH_INX:
3277 case RISCV::LHU:
3278 case RISCV::LW:
3279 case RISCV::LW_INX:
3280 case RISCV::LWU:
3281 case RISCV::LD:
3282 case RISCV::LD_RV32:
3283 case RISCV::FLH:
3284 case RISCV::FLW:
3285 case RISCV::FLD:
3286 case RISCV::SB:
3287 case RISCV::SH:
3288 case RISCV::SH_INX:
3289 case RISCV::SW:
3290 case RISCV::SW_INX:
3291 case RISCV::SD:
3292 case RISCV::SD_RV32:
3293 case RISCV::FSH:
3294 case RISCV::FSW:
3295 case RISCV::FSD:
3296 break;
3297 }
3298
3299 if (MemI.getOperand(i: 0).getReg() == Reg)
3300 return false;
3301
3302 if (AddrI.getOpcode() != RISCV::ADDI || !AddrI.getOperand(i: 1).isReg() ||
3303 !AddrI.getOperand(i: 2).isImm())
3304 return false;
3305
3306 int64_t OldOffset = MemI.getOperand(i: 2).getImm();
3307 int64_t Disp = AddrI.getOperand(i: 2).getImm();
3308 int64_t NewOffset = OldOffset + Disp;
3309 if (!STI.is64Bit())
3310 NewOffset = SignExtend64<32>(x: NewOffset);
3311
3312 if (!isInt<12>(x: NewOffset))
3313 return false;
3314
3315 AM.BaseReg = AddrI.getOperand(i: 1).getReg();
3316 AM.ScaledReg = 0;
3317 AM.Scale = 0;
3318 AM.Displacement = NewOffset;
3319 AM.Form = ExtAddrMode::Formula::Basic;
3320 return true;
3321}
3322
3323MachineInstr *RISCVInstrInfo::emitLdStWithAddr(MachineInstr &MemI,
3324 const ExtAddrMode &AM) const {
3325
3326 const DebugLoc &DL = MemI.getDebugLoc();
3327 MachineBasicBlock &MBB = *MemI.getParent();
3328
3329 assert(AM.ScaledReg == 0 && AM.Scale == 0 &&
3330 "Addressing mode not supported for folding");
3331
3332 return BuildMI(BB&: MBB, I&: MemI, MIMD: DL, MCID: get(Opcode: MemI.getOpcode()))
3333 .addReg(RegNo: MemI.getOperand(i: 0).getReg(), Flags: getDefRegState(B: MemI.mayLoad()))
3334 .addReg(RegNo: AM.BaseReg)
3335 .addImm(Val: AM.Displacement)
3336 .setMemRefs(MemI.memoperands())
3337 .setMIFlags(MemI.getFlags());
3338}
3339
3340// TODO: At the moment, MIPS introduced paring of instructions operating with
3341// word or double word. This should be extended with more instructions when more
3342// vendors support load/store pairing.
3343bool RISCVInstrInfo::isPairableLdStInstOpc(unsigned Opc) {
3344 switch (Opc) {
3345 default:
3346 return false;
3347 case RISCV::SW:
3348 case RISCV::SD:
3349 case RISCV::LD:
3350 case RISCV::LW:
3351 return true;
3352 }
3353}
3354
3355bool RISCVInstrInfo::isLdStSafeToPair(const MachineInstr &LdSt,
3356 const TargetRegisterInfo *TRI) {
3357 // If this is a volatile load/store, don't mess with it.
3358 if (LdSt.hasOrderedMemoryRef() || LdSt.getNumExplicitOperands() != 3)
3359 return false;
3360
3361 if (LdSt.getOperand(i: 1).isFI())
3362 return true;
3363
3364 assert(LdSt.getOperand(1).isReg() && "Expected a reg operand.");
3365 // Can't cluster if the instruction modifies the base register
3366 // or it is update form. e.g. ld x5,8(x5)
3367 if (LdSt.modifiesRegister(Reg: LdSt.getOperand(i: 1).getReg(), TRI))
3368 return false;
3369
3370 if (!LdSt.getOperand(i: 2).isImm())
3371 return false;
3372
3373 return true;
3374}
3375
3376bool RISCVInstrInfo::getMemOperandsWithOffsetWidth(
3377 const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
3378 int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
3379 const TargetRegisterInfo *TRI) const {
3380 if (!LdSt.mayLoadOrStore())
3381 return false;
3382
3383 // Conservatively, only handle scalar loads/stores for now.
3384 switch (LdSt.getOpcode()) {
3385 case RISCV::LB:
3386 case RISCV::LBU:
3387 case RISCV::SB:
3388 case RISCV::LH:
3389 case RISCV::LH_INX:
3390 case RISCV::LHU:
3391 case RISCV::FLH:
3392 case RISCV::SH:
3393 case RISCV::SH_INX:
3394 case RISCV::FSH:
3395 case RISCV::LW:
3396 case RISCV::LW_INX:
3397 case RISCV::LWU:
3398 case RISCV::FLW:
3399 case RISCV::SW:
3400 case RISCV::SW_INX:
3401 case RISCV::FSW:
3402 case RISCV::LD:
3403 case RISCV::LD_RV32:
3404 case RISCV::FLD:
3405 case RISCV::SD:
3406 case RISCV::SD_RV32:
3407 case RISCV::FSD:
3408 break;
3409 default:
3410 return false;
3411 }
3412 const MachineOperand *BaseOp;
3413 OffsetIsScalable = false;
3414 if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width, TRI))
3415 return false;
3416 BaseOps.push_back(Elt: BaseOp);
3417 return true;
3418}
3419
3420// TODO: This was copied from SIInstrInfo. Could it be lifted to a common
3421// helper?
3422static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
3423 ArrayRef<const MachineOperand *> BaseOps1,
3424 const MachineInstr &MI2,
3425 ArrayRef<const MachineOperand *> BaseOps2) {
3426 // Only examine the first "base" operand of each instruction, on the
3427 // assumption that it represents the real base address of the memory access.
3428 // Other operands are typically offsets or indices from this base address.
3429 if (BaseOps1.front()->isIdenticalTo(Other: *BaseOps2.front()))
3430 return true;
3431
3432 if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
3433 return false;
3434
3435 auto MO1 = *MI1.memoperands_begin();
3436 auto MO2 = *MI2.memoperands_begin();
3437 if (MO1->getAddrSpace() != MO2->getAddrSpace())
3438 return false;
3439
3440 auto Base1 = MO1->getValue();
3441 auto Base2 = MO2->getValue();
3442 if (!Base1 || !Base2)
3443 return false;
3444 Base1 = getUnderlyingObject(V: Base1);
3445 Base2 = getUnderlyingObject(V: Base2);
3446
3447 if (isa<UndefValue>(Val: Base1) || isa<UndefValue>(Val: Base2))
3448 return false;
3449
3450 return Base1 == Base2;
3451}
3452
3453bool RISCVInstrInfo::shouldClusterMemOps(
3454 ArrayRef<const MachineOperand *> BaseOps1, int64_t Offset1,
3455 bool OffsetIsScalable1, ArrayRef<const MachineOperand *> BaseOps2,
3456 int64_t Offset2, bool OffsetIsScalable2, unsigned ClusterSize,
3457 unsigned NumBytes) const {
3458 // If the mem ops (to be clustered) do not have the same base ptr, then they
3459 // should not be clustered
3460 if (!BaseOps1.empty() && !BaseOps2.empty()) {
3461 const MachineInstr &FirstLdSt = *BaseOps1.front()->getParent();
3462 const MachineInstr &SecondLdSt = *BaseOps2.front()->getParent();
3463 if (!memOpsHaveSameBasePtr(MI1: FirstLdSt, BaseOps1, MI2: SecondLdSt, BaseOps2))
3464 return false;
3465 } else if (!BaseOps1.empty() || !BaseOps2.empty()) {
3466 // If only one base op is empty, they do not have the same base ptr
3467 return false;
3468 }
3469
3470 unsigned CacheLineSize =
3471 BaseOps1.front()->getParent()->getMF()->getSubtarget().getCacheLineSize();
3472 // Assume a cache line size of 64 bytes if no size is set in RISCVSubtarget.
3473 CacheLineSize = CacheLineSize ? CacheLineSize : 64;
3474 // Cluster if the memory operations are on the same or a neighbouring cache
3475 // line, but limit the maximum ClusterSize to avoid creating too much
3476 // additional register pressure.
3477 return ClusterSize <= 4 && std::abs(i: Offset1 - Offset2) < CacheLineSize;
3478}
3479
3480// Set BaseReg (the base register operand), Offset (the byte offset being
3481// accessed) and the access Width of the passed instruction that reads/writes
3482// memory. Returns false if the instruction does not read/write memory or the
3483// BaseReg/Offset/Width can't be determined. Is not guaranteed to always
3484// recognise base operands and offsets in all cases.
3485// TODO: Add an IsScalable bool ref argument (like the equivalent AArch64
3486// function) and set it as appropriate.
3487bool RISCVInstrInfo::getMemOperandWithOffsetWidth(
3488 const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset,
3489 LocationSize &Width, const TargetRegisterInfo *TRI) const {
3490 if (!LdSt.mayLoadOrStore())
3491 return false;
3492
3493 // Here we assume the standard RISC-V ISA, which uses a base+offset
3494 // addressing mode. You'll need to relax these conditions to support custom
3495 // load/store instructions.
3496 if (LdSt.getNumExplicitOperands() != 3)
3497 return false;
3498 if ((!LdSt.getOperand(i: 1).isReg() && !LdSt.getOperand(i: 1).isFI()) ||
3499 !LdSt.getOperand(i: 2).isImm())
3500 return false;
3501
3502 if (!LdSt.hasOneMemOperand())
3503 return false;
3504
3505 Width = (*LdSt.memoperands_begin())->getSize();
3506 BaseReg = &LdSt.getOperand(i: 1);
3507 Offset = LdSt.getOperand(i: 2).getImm();
3508 return true;
3509}
3510
3511bool RISCVInstrInfo::areMemAccessesTriviallyDisjoint(
3512 const MachineInstr &MIa, const MachineInstr &MIb) const {
3513 assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
3514 assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
3515
3516 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() ||
3517 MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
3518 return false;
3519
3520 // Retrieve the base register, offset from the base register and width. Width
3521 // is the size of memory that is being loaded/stored (e.g. 1, 2, 4). If
3522 // base registers are identical, and the offset of a lower memory access +
3523 // the width doesn't overlap the offset of a higher memory access,
3524 // then the memory accesses are different.
3525 const TargetRegisterInfo *TRI = STI.getRegisterInfo();
3526 const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
3527 int64_t OffsetA = 0, OffsetB = 0;
3528 LocationSize WidthA = LocationSize::precise(Value: 0),
3529 WidthB = LocationSize::precise(Value: 0);
3530 if (getMemOperandWithOffsetWidth(LdSt: MIa, BaseReg&: BaseOpA, Offset&: OffsetA, Width&: WidthA, TRI) &&
3531 getMemOperandWithOffsetWidth(LdSt: MIb, BaseReg&: BaseOpB, Offset&: OffsetB, Width&: WidthB, TRI)) {
3532 if (BaseOpA->isIdenticalTo(Other: *BaseOpB)) {
3533 int LowOffset = std::min(a: OffsetA, b: OffsetB);
3534 int HighOffset = std::max(a: OffsetA, b: OffsetB);
3535 LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
3536 if (LowWidth.hasValue() &&
3537 LowOffset + (int)LowWidth.getValue() <= HighOffset)
3538 return true;
3539 }
3540 }
3541 return false;
3542}
3543
3544std::pair<unsigned, unsigned>
3545RISCVInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
3546 const unsigned Mask = RISCVII::MO_DIRECT_FLAG_MASK;
3547 return std::make_pair(x: TF & Mask, y: TF & ~Mask);
3548}
3549
3550ArrayRef<std::pair<unsigned, const char *>>
3551RISCVInstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
3552 using namespace RISCVII;
3553 static const std::pair<unsigned, const char *> TargetFlags[] = {
3554 {MO_CALL, "riscv-call"},
3555 {MO_LO, "riscv-lo"},
3556 {MO_HI, "riscv-hi"},
3557 {MO_PCREL_LO, "riscv-pcrel-lo"},
3558 {MO_PCREL_HI, "riscv-pcrel-hi"},
3559 {MO_GOT_HI, "riscv-got-hi"},
3560 {MO_TPREL_LO, "riscv-tprel-lo"},
3561 {MO_TPREL_HI, "riscv-tprel-hi"},
3562 {MO_TPREL_ADD, "riscv-tprel-add"},
3563 {MO_TLS_GOT_HI, "riscv-tls-got-hi"},
3564 {MO_TLS_GD_HI, "riscv-tls-gd-hi"},
3565 {MO_TLSDESC_HI, "riscv-tlsdesc-hi"},
3566 {MO_TLSDESC_LOAD_LO, "riscv-tlsdesc-load-lo"},
3567 {MO_TLSDESC_ADD_LO, "riscv-tlsdesc-add-lo"},
3568 {MO_TLSDESC_CALL, "riscv-tlsdesc-call"}};
3569 return ArrayRef(TargetFlags);
3570}
3571bool RISCVInstrInfo::isFunctionSafeToOutlineFrom(
3572 MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
3573 const Function &F = MF.getFunction();
3574
3575 // Can F be deduplicated by the linker? If it can, don't outline from it.
3576 if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage())
3577 return false;
3578
3579 // Don't outline from functions with section markings; the program could
3580 // expect that all the code is in the named section.
3581 if (F.hasSection())
3582 return false;
3583
3584 // It's safe to outline from MF.
3585 return true;
3586}
3587
3588bool RISCVInstrInfo::isMBBSafeToOutlineFrom(MachineBasicBlock &MBB,
3589 unsigned &Flags) const {
3590 // More accurate safety checking is done in getOutliningCandidateInfo.
3591 return TargetInstrInfo::isMBBSafeToOutlineFrom(MBB, Flags);
3592}
3593
3594// Enum values indicating how an outlined call should be constructed.
3595enum MachineOutlinerConstructionID {
3596 MachineOutlinerTailCall,
3597 MachineOutlinerDefault
3598};
3599
3600bool RISCVInstrInfo::shouldOutlineFromFunctionByDefault(
3601 MachineFunction &MF) const {
3602 return MF.getFunction().hasMinSize();
3603}
3604
3605static bool isCandidatePatchable(const MachineBasicBlock &MBB) {
3606 const MachineFunction *MF = MBB.getParent();
3607 const Function &F = MF->getFunction();
3608 return F.getFnAttribute(Kind: "fentry-call").getValueAsBool() ||
3609 F.hasFnAttribute(Kind: "patchable-function-entry");
3610}
3611
3612static bool isMIReadsReg(const MachineInstr &MI, const TargetRegisterInfo *TRI,
3613 MCRegister RegNo) {
3614 return MI.readsRegister(Reg: RegNo, TRI) ||
3615 MI.getDesc().hasImplicitUseOfPhysReg(Reg: RegNo);
3616}
3617
3618static bool isMIModifiesReg(const MachineInstr &MI,
3619 const TargetRegisterInfo *TRI, MCRegister RegNo) {
3620 return MI.modifiesRegister(Reg: RegNo, TRI) ||
3621 MI.getDesc().hasImplicitDefOfPhysReg(Reg: RegNo);
3622}
3623
3624static bool cannotInsertTailCall(const MachineBasicBlock &MBB) {
3625 if (!MBB.back().isReturn())
3626 return true;
3627 if (isCandidatePatchable(MBB))
3628 return true;
3629
3630 // If the candidate reads the pre-set register
3631 // that can be used for expanding PseudoTAIL instruction,
3632 // then we cannot insert tail call.
3633 const TargetSubtargetInfo &STI = MBB.getParent()->getSubtarget();
3634 MCRegister TailExpandUseRegNo =
3635 RISCVII::getTailExpandUseRegNo(FeatureBits: STI.getFeatureBits());
3636 for (const MachineInstr &MI : MBB) {
3637 if (isMIReadsReg(MI, TRI: STI.getRegisterInfo(), RegNo: TailExpandUseRegNo))
3638 return true;
3639 if (isMIModifiesReg(MI, TRI: STI.getRegisterInfo(), RegNo: TailExpandUseRegNo))
3640 break;
3641 }
3642 return false;
3643}
3644
3645bool RISCVInstrInfo::analyzeCandidate(outliner::Candidate &C) const {
3646 // If the expansion register for tail calls is live across the candidate
3647 // outlined call site, we cannot outline that candidate as the expansion
3648 // would clobber the register.
3649 MCRegister TailExpandUseReg =
3650 RISCVII::getTailExpandUseRegNo(FeatureBits: STI.getFeatureBits());
3651 if (C.back().isReturn() &&
3652 !C.isAvailableAcrossAndOutOfSeq(Reg: TailExpandUseReg, TRI: RegInfo)) {
3653 LLVM_DEBUG(dbgs() << "MBB:\n" << *C.getMBB());
3654 LLVM_DEBUG(dbgs() << "Cannot be outlined between: " << C.front() << "and "
3655 << C.back());
3656 LLVM_DEBUG(dbgs() << "Because the tail-call register is live across "
3657 "the proposed outlined function call\n");
3658 return true;
3659 }
3660
3661 // If last instruction is return then we can rely on
3662 // the verification already performed in the getOutliningTypeImpl.
3663 if (C.back().isReturn()) {
3664 assert(!cannotInsertTailCall(*C.getMBB()) &&
3665 "The candidate who uses return instruction must be outlined "
3666 "using tail call");
3667 return false;
3668 }
3669
3670 // Filter out candidates where the X5 register (t0) can't be used to setup
3671 // the function call.
3672 if (llvm::any_of(Range&: C, P: [this](const MachineInstr &MI) {
3673 return isMIModifiesReg(MI, TRI: &RegInfo, RegNo: RISCV::X5);
3674 }))
3675 return true;
3676
3677 return !C.isAvailableAcrossAndOutOfSeq(Reg: RISCV::X5, TRI: RegInfo);
3678}
3679
3680std::optional<std::unique_ptr<outliner::OutlinedFunction>>
3681RISCVInstrInfo::getOutliningCandidateInfo(
3682 const MachineModuleInfo &MMI,
3683 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
3684 unsigned MinRepeats) const {
3685
3686 // Analyze each candidate and erase the ones that are not viable.
3687 llvm::erase_if(C&: RepeatedSequenceLocs, P: [this](auto Candidate) {
3688 return analyzeCandidate(C&: Candidate);
3689 });
3690
3691 // If the sequence doesn't have enough candidates left, then we're done.
3692 if (RepeatedSequenceLocs.size() < MinRepeats)
3693 return std::nullopt;
3694
3695 // Each RepeatedSequenceLoc is identical.
3696 outliner::Candidate &Candidate = RepeatedSequenceLocs[0];
3697 unsigned InstrSizeCExt =
3698 Candidate.getMF()->getSubtarget<RISCVSubtarget>().hasStdExtZca() ? 2 : 4;
3699 unsigned CallOverhead = 0, FrameOverhead = 0;
3700
3701 // Count the number of CFI instructions in the candidate, if present.
3702 unsigned CFICount = 0;
3703 for (auto &I : Candidate) {
3704 if (I.isCFIInstruction())
3705 CFICount++;
3706 }
3707
3708 // Ensure CFI coverage matches: comparing the number of CFIs in the candidate
3709 // with the total number of CFIs in the parent function for each candidate.
3710 // Outlining only a subset of a function’s CFIs would split the unwind state
3711 // across two code regions and lead to incorrect address offsets between the
3712 // outlined body and the remaining code. To preserve correct unwind info, we
3713 // only outline when all CFIs in the function can be outlined together.
3714 for (outliner::Candidate &C : RepeatedSequenceLocs) {
3715 std::vector<MCCFIInstruction> CFIInstructions =
3716 C.getMF()->getFrameInstructions();
3717
3718 if (CFICount > 0 && CFICount != CFIInstructions.size())
3719 return std::nullopt;
3720 }
3721
3722 MachineOutlinerConstructionID MOCI = MachineOutlinerDefault;
3723 if (Candidate.back().isReturn()) {
3724 MOCI = MachineOutlinerTailCall;
3725 // tail call = auipc + jalr in the worst case without linker relaxation.
3726 // FIXME: This code suggests the JALR can be compressed - how?
3727 CallOverhead = 4 + InstrSizeCExt;
3728 // Using tail call we move ret instruction from caller to callee.
3729 FrameOverhead = 0;
3730 } else {
3731 // call t0, function = 8 bytes.
3732 CallOverhead = 8;
3733 // jr t0 = 4 bytes, 2 bytes if compressed instructions are enabled.
3734 FrameOverhead = InstrSizeCExt;
3735 }
3736
3737 // If we have CFI instructions, we can only outline if the outlined section
3738 // can be a tail call.
3739 if (MOCI != MachineOutlinerTailCall && CFICount > 0)
3740 return std::nullopt;
3741
3742 for (auto &C : RepeatedSequenceLocs)
3743 C.setCallInfo(CID: MOCI, CO: CallOverhead);
3744
3745 unsigned SequenceSize = 0;
3746 for (auto &MI : Candidate)
3747 SequenceSize += getInstSizeInBytes(MI);
3748
3749 return std::make_unique<outliner::OutlinedFunction>(
3750 args&: RepeatedSequenceLocs, args&: SequenceSize, args&: FrameOverhead, args&: MOCI);
3751}
3752
3753outliner::InstrType
3754RISCVInstrInfo::getOutliningTypeImpl(const MachineModuleInfo &MMI,
3755 MachineBasicBlock::iterator &MBBI,
3756 unsigned Flags) const {
3757 MachineInstr &MI = *MBBI;
3758 MachineBasicBlock *MBB = MI.getParent();
3759 const TargetRegisterInfo *TRI =
3760 MBB->getParent()->getSubtarget().getRegisterInfo();
3761 const auto &F = MI.getMF()->getFunction();
3762
3763 // We can only outline CFI instructions if we will tail call the outlined
3764 // function, or fix up the CFI offsets. Currently, CFI instructions are
3765 // outlined only if in a tail call.
3766 if (MI.isCFIInstruction())
3767 return outliner::InstrType::Legal;
3768
3769 if (cannotInsertTailCall(MBB: *MBB) &&
3770 (MI.isReturn() || isMIModifiesReg(MI, TRI, RegNo: RISCV::X5)))
3771 return outliner::InstrType::Illegal;
3772
3773 // Make sure the operands don't reference something unsafe.
3774 for (const auto &MO : MI.operands()) {
3775
3776 // pcrel-hi and pcrel-lo can't put in separate sections, filter that out
3777 // if any possible.
3778 if (MO.getTargetFlags() == RISCVII::MO_PCREL_LO &&
3779 (MI.getMF()->getTarget().getFunctionSections() || F.hasComdat() ||
3780 F.hasSection() || F.getSectionPrefix()))
3781 return outliner::InstrType::Illegal;
3782 }
3783
3784 if (isLPAD(MI))
3785 return outliner::InstrType::Illegal;
3786
3787 return outliner::InstrType::Legal;
3788}
3789
3790void RISCVInstrInfo::buildOutlinedFrame(
3791 MachineBasicBlock &MBB, MachineFunction &MF,
3792 const outliner::OutlinedFunction &OF) const {
3793
3794 if (OF.FrameConstructionID == MachineOutlinerTailCall)
3795 return;
3796
3797 MBB.addLiveIn(PhysReg: RISCV::X5);
3798
3799 // Add in a return instruction to the end of the outlined frame.
3800 MBB.insert(I: MBB.end(), MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: RISCV::JALR))
3801 .addReg(RegNo: RISCV::X0, Flags: RegState::Define)
3802 .addReg(RegNo: RISCV::X5)
3803 .addImm(Val: 0));
3804}
3805
3806MachineBasicBlock::iterator RISCVInstrInfo::insertOutlinedCall(
3807 Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It,
3808 MachineFunction &MF, outliner::Candidate &C) const {
3809
3810 if (C.CallConstructionID == MachineOutlinerTailCall) {
3811 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: RISCV::PseudoTAIL))
3812 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName()),
3813 /*Offset=*/0, TargetFlags: RISCVII::MO_CALL));
3814 return It;
3815 }
3816
3817 // Add in a call instruction to the outlined function at the given location.
3818 It = MBB.insert(I: It,
3819 MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: RISCV::PseudoCALLReg), DestReg: RISCV::X5)
3820 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName()), Offset: 0,
3821 TargetFlags: RISCVII::MO_CALL));
3822 return It;
3823}
3824
3825std::optional<RegImmPair> RISCVInstrInfo::isAddImmediate(const MachineInstr &MI,
3826 Register Reg) const {
3827 // TODO: Handle cases where Reg is a super- or sub-register of the
3828 // destination register.
3829 const MachineOperand &Op0 = MI.getOperand(i: 0);
3830 if (!Op0.isReg() || Reg != Op0.getReg())
3831 return std::nullopt;
3832
3833 // Don't consider ADDIW as a candidate because the caller may not be aware
3834 // of its sign extension behaviour.
3835 if (MI.getOpcode() == RISCV::ADDI && MI.getOperand(i: 1).isReg() &&
3836 MI.getOperand(i: 2).isImm())
3837 return RegImmPair{MI.getOperand(i: 1).getReg(), MI.getOperand(i: 2).getImm()};
3838
3839 return std::nullopt;
3840}
3841
3842// MIR printer helper function to annotate Operands with a comment.
3843std::string RISCVInstrInfo::createMIROperandComment(
3844 const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx,
3845 const TargetRegisterInfo *TRI) const {
3846 // Print a generic comment for this operand if there is one.
3847 std::string GenericComment =
3848 TargetInstrInfo::createMIROperandComment(MI, Op, OpIdx, TRI);
3849 if (!GenericComment.empty())
3850 return GenericComment;
3851
3852 // If not, we must have an immediate operand.
3853 if (!Op.isImm())
3854 return std::string();
3855
3856 const MCInstrDesc &Desc = MI.getDesc();
3857 if (OpIdx >= Desc.getNumOperands())
3858 return std::string();
3859
3860 std::string Comment;
3861 raw_string_ostream OS(Comment);
3862
3863 const MCOperandInfo &OpInfo = Desc.operands()[OpIdx];
3864
3865 // Print the full VType operand of vsetvli/vsetivli instructions, and the SEW
3866 // operand of vector codegen pseudos.
3867 switch (OpInfo.OperandType) {
3868 case RISCVOp::OPERAND_VTYPEI10:
3869 case RISCVOp::OPERAND_VTYPEI11: {
3870 unsigned Imm = Op.getImm();
3871 RISCVVType::printVType(VType: Imm, OS);
3872 break;
3873 }
3874 case RISCVOp::OPERAND_XSFMM_VTYPE: {
3875 unsigned Imm = Op.getImm();
3876 RISCVVType::printXSfmmVType(VType: Imm, OS);
3877 break;
3878 }
3879 case RISCVOp::OPERAND_XSFMM_TWIDEN: {
3880 unsigned Imm = Op.getImm();
3881 OS << "w" << Imm;
3882 break;
3883 }
3884 case RISCVOp::OPERAND_SEW:
3885 case RISCVOp::OPERAND_SEW_MASK: {
3886 unsigned Log2SEW = Op.getImm();
3887 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
3888 assert(RISCVVType::isValidSEW(SEW) && "Unexpected SEW");
3889 OS << "e" << SEW;
3890 break;
3891 }
3892 case RISCVOp::OPERAND_VEC_POLICY:
3893 unsigned Policy = Op.getImm();
3894 assert(Policy <= (RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC) &&
3895 "Invalid Policy Value");
3896 OS << (Policy & RISCVVType::TAIL_AGNOSTIC ? "ta" : "tu") << ", "
3897 << (Policy & RISCVVType::MASK_AGNOSTIC ? "ma" : "mu");
3898 break;
3899 }
3900
3901 return Comment;
3902}
3903
3904// clang-format off
3905#define CASE_RVV_OPCODE_UNMASK_LMUL(OP, LMUL) \
3906 RISCV::Pseudo##OP##_##LMUL
3907
3908#define CASE_RVV_OPCODE_MASK_LMUL(OP, LMUL) \
3909 RISCV::Pseudo##OP##_##LMUL##_MASK
3910
3911#define CASE_RVV_OPCODE_LMUL(OP, LMUL) \
3912 CASE_RVV_OPCODE_UNMASK_LMUL(OP, LMUL): \
3913 case CASE_RVV_OPCODE_MASK_LMUL(OP, LMUL)
3914
3915#define CASE_RVV_OPCODE_UNMASK_WIDEN(OP) \
3916 CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF8): \
3917 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF4): \
3918 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF2): \
3919 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M1): \
3920 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M2): \
3921 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M4)
3922
3923#define CASE_RVV_OPCODE_UNMASK(OP) \
3924 CASE_RVV_OPCODE_UNMASK_WIDEN(OP): \
3925 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M8)
3926
3927#define CASE_RVV_OPCODE_MASK_WIDEN(OP) \
3928 CASE_RVV_OPCODE_MASK_LMUL(OP, MF8): \
3929 case CASE_RVV_OPCODE_MASK_LMUL(OP, MF4): \
3930 case CASE_RVV_OPCODE_MASK_LMUL(OP, MF2): \
3931 case CASE_RVV_OPCODE_MASK_LMUL(OP, M1): \
3932 case CASE_RVV_OPCODE_MASK_LMUL(OP, M2): \
3933 case CASE_RVV_OPCODE_MASK_LMUL(OP, M4)
3934
3935#define CASE_RVV_OPCODE_MASK(OP) \
3936 CASE_RVV_OPCODE_MASK_WIDEN(OP): \
3937 case CASE_RVV_OPCODE_MASK_LMUL(OP, M8)
3938
3939#define CASE_RVV_OPCODE_WIDEN(OP) \
3940 CASE_RVV_OPCODE_UNMASK_WIDEN(OP): \
3941 case CASE_RVV_OPCODE_MASK_WIDEN(OP)
3942
3943#define CASE_RVV_OPCODE(OP) \
3944 CASE_RVV_OPCODE_UNMASK(OP): \
3945 case CASE_RVV_OPCODE_MASK(OP)
3946// clang-format on
3947
3948// clang-format off
3949#define CASE_VMA_OPCODE_COMMON(OP, TYPE, LMUL) \
3950 RISCV::PseudoV##OP##_##TYPE##_##LMUL
3951
3952#define CASE_VMA_OPCODE_LMULS(OP, TYPE) \
3953 CASE_VMA_OPCODE_COMMON(OP, TYPE, MF8): \
3954 case CASE_VMA_OPCODE_COMMON(OP, TYPE, MF4): \
3955 case CASE_VMA_OPCODE_COMMON(OP, TYPE, MF2): \
3956 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M1): \
3957 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M2): \
3958 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M4): \
3959 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M8)
3960
3961// VFMA instructions are SEW specific.
3962#define CASE_VFMA_OPCODE_COMMON(OP, TYPE, LMUL, SEW) \
3963 RISCV::PseudoV##OP##_##TYPE##_##LMUL##_##SEW
3964
3965#define CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE, SEW) \
3966 CASE_VFMA_OPCODE_COMMON(OP, TYPE, M1, SEW): \
3967 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M2, SEW): \
3968 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M4, SEW): \
3969 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M8, SEW)
3970
3971#define CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE, SEW) \
3972 CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF2, SEW): \
3973 case CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE, SEW)
3974
3975#define CASE_VFMA_OPCODE_LMULS_MF4(OP, TYPE, SEW) \
3976 CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF4, SEW): \
3977 case CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE, SEW)
3978
3979#define CASE_VFMA_OPCODE_VV(OP) \
3980 CASE_VFMA_OPCODE_LMULS_MF4(OP, VV, E16): \
3981 case CASE_VFMA_OPCODE_LMULS_MF4(OP##_ALT, VV, E16): \
3982 case CASE_VFMA_OPCODE_LMULS_MF2(OP, VV, E32): \
3983 case CASE_VFMA_OPCODE_LMULS_M1(OP, VV, E64)
3984
3985#define CASE_VFMA_SPLATS(OP) \
3986 CASE_VFMA_OPCODE_LMULS_MF4(OP, VFPR16, E16): \
3987 case CASE_VFMA_OPCODE_LMULS_MF4(OP##_ALT, VFPR16, E16): \
3988 case CASE_VFMA_OPCODE_LMULS_MF2(OP, VFPR32, E32): \
3989 case CASE_VFMA_OPCODE_LMULS_M1(OP, VFPR64, E64)
3990// clang-format on
3991
3992bool RISCVInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
3993 unsigned &SrcOpIdx1,
3994 unsigned &SrcOpIdx2) const {
3995 const MCInstrDesc &Desc = MI.getDesc();
3996 if (!Desc.isCommutable())
3997 return false;
3998
3999 switch (MI.getOpcode()) {
4000 case RISCV::TH_MVEQZ:
4001 case RISCV::TH_MVNEZ:
4002 // We can't commute operands if operand 2 (i.e., rs1 in
4003 // mveqz/mvnez rd,rs1,rs2) is the zero-register (as it is
4004 // not valid as the in/out-operand 1).
4005 if (MI.getOperand(i: 2).getReg() == RISCV::X0)
4006 return false;
4007 // Operands 1 and 2 are commutable, if we switch the opcode.
4008 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 1, CommutableOpIdx2: 2);
4009 case RISCV::QC_SELECTIEQ:
4010 case RISCV::QC_SELECTINE:
4011 case RISCV::QC_SELECTIIEQ:
4012 case RISCV::QC_SELECTIINE:
4013 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 1, CommutableOpIdx2: 2);
4014 case RISCV::QC_MVEQ:
4015 case RISCV::QC_MVNE:
4016 case RISCV::QC_MVLT:
4017 case RISCV::QC_MVGE:
4018 case RISCV::QC_MVLTU:
4019 case RISCV::QC_MVGEU:
4020 case RISCV::QC_MVEQI:
4021 case RISCV::QC_MVNEI:
4022 case RISCV::QC_MVLTI:
4023 case RISCV::QC_MVGEI:
4024 case RISCV::QC_MVLTUI:
4025 case RISCV::QC_MVGEUI:
4026 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 1, CommutableOpIdx2: 4);
4027 case RISCV::TH_MULA:
4028 case RISCV::TH_MULAW:
4029 case RISCV::TH_MULAH:
4030 case RISCV::TH_MULS:
4031 case RISCV::TH_MULSW:
4032 case RISCV::TH_MULSH:
4033 // Operands 2 and 3 are commutable.
4034 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 2, CommutableOpIdx2: 3);
4035 case RISCV::PseudoCCMOVGPRNoX0:
4036 case RISCV::PseudoCCMOVGPR:
4037 // Operands 4 and 5 are commutable.
4038 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 4, CommutableOpIdx2: 5);
4039 case CASE_RVV_OPCODE(VADD_VV):
4040 case CASE_RVV_OPCODE(VAND_VV):
4041 case CASE_RVV_OPCODE(VOR_VV):
4042 case CASE_RVV_OPCODE(VXOR_VV):
4043 case CASE_RVV_OPCODE_MASK(VMSEQ_VV):
4044 case CASE_RVV_OPCODE_MASK(VMSNE_VV):
4045 case CASE_RVV_OPCODE(VMIN_VV):
4046 case CASE_RVV_OPCODE(VMINU_VV):
4047 case CASE_RVV_OPCODE(VMAX_VV):
4048 case CASE_RVV_OPCODE(VMAXU_VV):
4049 case CASE_RVV_OPCODE(VMUL_VV):
4050 case CASE_RVV_OPCODE(VMULH_VV):
4051 case CASE_RVV_OPCODE(VMULHU_VV):
4052 case CASE_RVV_OPCODE_WIDEN(VWADD_VV):
4053 case CASE_RVV_OPCODE_WIDEN(VWADDU_VV):
4054 case CASE_RVV_OPCODE_WIDEN(VWMUL_VV):
4055 case CASE_RVV_OPCODE_WIDEN(VWMULU_VV):
4056 case CASE_RVV_OPCODE_WIDEN(VWMACC_VV):
4057 case CASE_RVV_OPCODE_WIDEN(VWMACCU_VV):
4058 case CASE_RVV_OPCODE_UNMASK(VADC_VVM):
4059 case CASE_RVV_OPCODE(VSADD_VV):
4060 case CASE_RVV_OPCODE(VSADDU_VV):
4061 case CASE_RVV_OPCODE(VAADD_VV):
4062 case CASE_RVV_OPCODE(VAADDU_VV):
4063 case CASE_RVV_OPCODE(VSMUL_VV):
4064 // Operands 2 and 3 are commutable.
4065 return fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1: 2, CommutableOpIdx2: 3);
4066 case CASE_VFMA_SPLATS(FMADD):
4067 case CASE_VFMA_SPLATS(FMSUB):
4068 case CASE_VFMA_SPLATS(FMACC):
4069 case CASE_VFMA_SPLATS(FMSAC):
4070 case CASE_VFMA_SPLATS(FNMADD):
4071 case CASE_VFMA_SPLATS(FNMSUB):
4072 case CASE_VFMA_SPLATS(FNMACC):
4073 case CASE_VFMA_SPLATS(FNMSAC):
4074 case CASE_VFMA_OPCODE_VV(FMACC):
4075 case CASE_VFMA_OPCODE_VV(FMSAC):
4076 case CASE_VFMA_OPCODE_VV(FNMACC):
4077 case CASE_VFMA_OPCODE_VV(FNMSAC):
4078 case CASE_VMA_OPCODE_LMULS(MADD, VX):
4079 case CASE_VMA_OPCODE_LMULS(NMSUB, VX):
4080 case CASE_VMA_OPCODE_LMULS(MACC, VX):
4081 case CASE_VMA_OPCODE_LMULS(NMSAC, VX):
4082 case CASE_VMA_OPCODE_LMULS(MACC, VV):
4083 case CASE_VMA_OPCODE_LMULS(NMSAC, VV): {
4084 // If the tail policy is undisturbed we can't commute.
4085 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags));
4086 if ((MI.getOperand(i: RISCVII::getVecPolicyOpNum(Desc: MI.getDesc())).getImm() &
4087 1) == 0)
4088 return false;
4089
4090 // For these instructions we can only swap operand 1 and operand 3 by
4091 // changing the opcode.
4092 unsigned CommutableOpIdx1 = 1;
4093 unsigned CommutableOpIdx2 = 3;
4094 if (!fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1,
4095 CommutableOpIdx2))
4096 return false;
4097 return true;
4098 }
4099 case CASE_VFMA_OPCODE_VV(FMADD):
4100 case CASE_VFMA_OPCODE_VV(FMSUB):
4101 case CASE_VFMA_OPCODE_VV(FNMADD):
4102 case CASE_VFMA_OPCODE_VV(FNMSUB):
4103 case CASE_VMA_OPCODE_LMULS(MADD, VV):
4104 case CASE_VMA_OPCODE_LMULS(NMSUB, VV): {
4105 // If the tail policy is undisturbed we can't commute.
4106 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags));
4107 if ((MI.getOperand(i: RISCVII::getVecPolicyOpNum(Desc: MI.getDesc())).getImm() &
4108 1) == 0)
4109 return false;
4110
4111 // For these instructions we have more freedom. We can commute with the
4112 // other multiplicand or with the addend/subtrahend/minuend.
4113
4114 // Any fixed operand must be from source 1, 2 or 3.
4115 if (SrcOpIdx1 != CommuteAnyOperandIndex && SrcOpIdx1 > 3)
4116 return false;
4117 if (SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx2 > 3)
4118 return false;
4119
4120 // It both ops are fixed one must be the tied source.
4121 if (SrcOpIdx1 != CommuteAnyOperandIndex &&
4122 SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx1 != 1 && SrcOpIdx2 != 1)
4123 return false;
4124
4125 // Look for two different register operands assumed to be commutable
4126 // regardless of the FMA opcode. The FMA opcode is adjusted later if
4127 // needed.
4128 if (SrcOpIdx1 == CommuteAnyOperandIndex ||
4129 SrcOpIdx2 == CommuteAnyOperandIndex) {
4130 // At least one of operands to be commuted is not specified and
4131 // this method is free to choose appropriate commutable operands.
4132 unsigned CommutableOpIdx1 = SrcOpIdx1;
4133 if (SrcOpIdx1 == SrcOpIdx2) {
4134 // Both of operands are not fixed. Set one of commutable
4135 // operands to the tied source.
4136 CommutableOpIdx1 = 1;
4137 } else if (SrcOpIdx1 == CommuteAnyOperandIndex) {
4138 // Only one of the operands is not fixed.
4139 CommutableOpIdx1 = SrcOpIdx2;
4140 }
4141
4142 // CommutableOpIdx1 is well defined now. Let's choose another commutable
4143 // operand and assign its index to CommutableOpIdx2.
4144 unsigned CommutableOpIdx2;
4145 if (CommutableOpIdx1 != 1) {
4146 // If we haven't already used the tied source, we must use it now.
4147 CommutableOpIdx2 = 1;
4148 } else {
4149 Register Op1Reg = MI.getOperand(i: CommutableOpIdx1).getReg();
4150
4151 // The commuted operands should have different registers.
4152 // Otherwise, the commute transformation does not change anything and
4153 // is useless. We use this as a hint to make our decision.
4154 if (Op1Reg != MI.getOperand(i: 2).getReg())
4155 CommutableOpIdx2 = 2;
4156 else
4157 CommutableOpIdx2 = 3;
4158 }
4159
4160 // Assign the found pair of commutable indices to SrcOpIdx1 and
4161 // SrcOpIdx2 to return those values.
4162 if (!fixCommutedOpIndices(ResultIdx1&: SrcOpIdx1, ResultIdx2&: SrcOpIdx2, CommutableOpIdx1,
4163 CommutableOpIdx2))
4164 return false;
4165 }
4166
4167 return true;
4168 }
4169 }
4170
4171 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
4172}
4173
4174// clang-format off
4175#define CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL) \
4176 case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL: \
4177 Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL; \
4178 break;
4179
4180#define CASE_VMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE) \
4181 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF8) \
4182 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4) \
4183 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2) \
4184 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1) \
4185 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2) \
4186 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4) \
4187 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8)
4188
4189// VFMA depends on SEW.
4190#define CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL, SEW) \
4191 case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL##_##SEW: \
4192 Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL##_##SEW; \
4193 break;
4194
4195#define CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE, SEW) \
4196 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1, SEW) \
4197 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2, SEW) \
4198 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4, SEW) \
4199 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8, SEW)
4200
4201#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE, SEW) \
4202 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2, SEW) \
4203 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE, SEW)
4204
4205#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE, SEW) \
4206 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4, SEW) \
4207 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE, SEW)
4208
4209#define CASE_VFMA_CHANGE_OPCODE_VV(OLDOP, NEWOP) \
4210 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VV, E16) \
4211 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP##_ALT, NEWOP##_ALT, VV, E16) \
4212 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VV, E32) \
4213 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VV, E64)
4214
4215#define CASE_VFMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP) \
4216 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VFPR16, E16) \
4217 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP##_ALT, NEWOP##_ALT, VFPR16, E16) \
4218 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VFPR32, E32) \
4219 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VFPR64, E64)
4220// clang-format on
4221
4222MachineInstr *RISCVInstrInfo::commuteInstructionImpl(MachineInstr &MI,
4223 bool NewMI,
4224 unsigned OpIdx1,
4225 unsigned OpIdx2) const {
4226 auto cloneIfNew = [NewMI](MachineInstr &MI) -> MachineInstr & {
4227 if (NewMI)
4228 return *MI.getParent()->getParent()->CloneMachineInstr(Orig: &MI);
4229 return MI;
4230 };
4231
4232 switch (MI.getOpcode()) {
4233 case RISCV::TH_MVEQZ:
4234 case RISCV::TH_MVNEZ: {
4235 auto &WorkingMI = cloneIfNew(MI);
4236 WorkingMI.setDesc(get(Opcode: MI.getOpcode() == RISCV::TH_MVEQZ ? RISCV::TH_MVNEZ
4237 : RISCV::TH_MVEQZ));
4238 return TargetInstrInfo::commuteInstructionImpl(MI&: WorkingMI, NewMI: false, OpIdx1,
4239 OpIdx2);
4240 }
4241 case RISCV::QC_SELECTIEQ:
4242 case RISCV::QC_SELECTINE:
4243 case RISCV::QC_SELECTIIEQ:
4244 case RISCV::QC_SELECTIINE:
4245 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
4246 case RISCV::QC_MVEQ:
4247 case RISCV::QC_MVNE:
4248 case RISCV::QC_MVLT:
4249 case RISCV::QC_MVGE:
4250 case RISCV::QC_MVLTU:
4251 case RISCV::QC_MVGEU:
4252 case RISCV::QC_MVEQI:
4253 case RISCV::QC_MVNEI:
4254 case RISCV::QC_MVLTI:
4255 case RISCV::QC_MVGEI:
4256 case RISCV::QC_MVLTUI:
4257 case RISCV::QC_MVGEUI: {
4258 auto &WorkingMI = cloneIfNew(MI);
4259 WorkingMI.setDesc(get(Opcode: getInverseXqcicmOpcode(Opcode: MI.getOpcode())));
4260 return TargetInstrInfo::commuteInstructionImpl(MI&: WorkingMI, NewMI: false, OpIdx1,
4261 OpIdx2);
4262 }
4263 case RISCV::PseudoCCMOVGPRNoX0:
4264 case RISCV::PseudoCCMOVGPR: {
4265 // CCMOV can be commuted by inverting the condition.
4266 auto CC = static_cast<RISCVCC::CondCode>(MI.getOperand(i: 3).getImm());
4267 CC = RISCVCC::getInverseBranchCondition(CC);
4268 auto &WorkingMI = cloneIfNew(MI);
4269 WorkingMI.getOperand(i: 3).setImm(CC);
4270 return TargetInstrInfo::commuteInstructionImpl(MI&: WorkingMI, /*NewMI*/ false,
4271 OpIdx1, OpIdx2);
4272 }
4273 case CASE_VFMA_SPLATS(FMACC):
4274 case CASE_VFMA_SPLATS(FMADD):
4275 case CASE_VFMA_SPLATS(FMSAC):
4276 case CASE_VFMA_SPLATS(FMSUB):
4277 case CASE_VFMA_SPLATS(FNMACC):
4278 case CASE_VFMA_SPLATS(FNMADD):
4279 case CASE_VFMA_SPLATS(FNMSAC):
4280 case CASE_VFMA_SPLATS(FNMSUB):
4281 case CASE_VFMA_OPCODE_VV(FMACC):
4282 case CASE_VFMA_OPCODE_VV(FMSAC):
4283 case CASE_VFMA_OPCODE_VV(FNMACC):
4284 case CASE_VFMA_OPCODE_VV(FNMSAC):
4285 case CASE_VMA_OPCODE_LMULS(MADD, VX):
4286 case CASE_VMA_OPCODE_LMULS(NMSUB, VX):
4287 case CASE_VMA_OPCODE_LMULS(MACC, VX):
4288 case CASE_VMA_OPCODE_LMULS(NMSAC, VX):
4289 case CASE_VMA_OPCODE_LMULS(MACC, VV):
4290 case CASE_VMA_OPCODE_LMULS(NMSAC, VV): {
4291 // It only make sense to toggle these between clobbering the
4292 // addend/subtrahend/minuend one of the multiplicands.
4293 assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");
4294 assert((OpIdx1 == 3 || OpIdx2 == 3) && "Unexpected opcode index");
4295 unsigned Opc;
4296 switch (MI.getOpcode()) {
4297 default:
4298 llvm_unreachable("Unexpected opcode");
4299 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMACC, FMADD)
4300 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMADD, FMACC)
4301 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMSAC, FMSUB)
4302 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMSUB, FMSAC)
4303 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMACC, FNMADD)
4304 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMADD, FNMACC)
4305 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMSAC, FNMSUB)
4306 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMSUB, FNMSAC)
4307 CASE_VFMA_CHANGE_OPCODE_VV(FMACC, FMADD)
4308 CASE_VFMA_CHANGE_OPCODE_VV(FMSAC, FMSUB)
4309 CASE_VFMA_CHANGE_OPCODE_VV(FNMACC, FNMADD)
4310 CASE_VFMA_CHANGE_OPCODE_VV(FNMSAC, FNMSUB)
4311 CASE_VMA_CHANGE_OPCODE_LMULS(MACC, MADD, VX)
4312 CASE_VMA_CHANGE_OPCODE_LMULS(MADD, MACC, VX)
4313 CASE_VMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VX)
4314 CASE_VMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VX)
4315 CASE_VMA_CHANGE_OPCODE_LMULS(MACC, MADD, VV)
4316 CASE_VMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VV)
4317 }
4318
4319 auto &WorkingMI = cloneIfNew(MI);
4320 WorkingMI.setDesc(get(Opcode: Opc));
4321 return TargetInstrInfo::commuteInstructionImpl(MI&: WorkingMI, /*NewMI=*/false,
4322 OpIdx1, OpIdx2);
4323 }
4324 case CASE_VFMA_OPCODE_VV(FMADD):
4325 case CASE_VFMA_OPCODE_VV(FMSUB):
4326 case CASE_VFMA_OPCODE_VV(FNMADD):
4327 case CASE_VFMA_OPCODE_VV(FNMSUB):
4328 case CASE_VMA_OPCODE_LMULS(MADD, VV):
4329 case CASE_VMA_OPCODE_LMULS(NMSUB, VV): {
4330 assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");
4331 // If one of the operands, is the addend we need to change opcode.
4332 // Otherwise we're just swapping 2 of the multiplicands.
4333 if (OpIdx1 == 3 || OpIdx2 == 3) {
4334 unsigned Opc;
4335 switch (MI.getOpcode()) {
4336 default:
4337 llvm_unreachable("Unexpected opcode");
4338 CASE_VFMA_CHANGE_OPCODE_VV(FMADD, FMACC)
4339 CASE_VFMA_CHANGE_OPCODE_VV(FMSUB, FMSAC)
4340 CASE_VFMA_CHANGE_OPCODE_VV(FNMADD, FNMACC)
4341 CASE_VFMA_CHANGE_OPCODE_VV(FNMSUB, FNMSAC)
4342 CASE_VMA_CHANGE_OPCODE_LMULS(MADD, MACC, VV)
4343 CASE_VMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VV)
4344 }
4345
4346 auto &WorkingMI = cloneIfNew(MI);
4347 WorkingMI.setDesc(get(Opcode: Opc));
4348 return TargetInstrInfo::commuteInstructionImpl(MI&: WorkingMI, /*NewMI=*/false,
4349 OpIdx1, OpIdx2);
4350 }
4351 // Let the default code handle it.
4352 break;
4353 }
4354 }
4355
4356 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
4357}
4358
4359#undef CASE_VMA_CHANGE_OPCODE_COMMON
4360#undef CASE_VMA_CHANGE_OPCODE_LMULS
4361#undef CASE_VFMA_CHANGE_OPCODE_COMMON
4362#undef CASE_VFMA_CHANGE_OPCODE_LMULS_M1
4363#undef CASE_VFMA_CHANGE_OPCODE_LMULS_MF2
4364#undef CASE_VFMA_CHANGE_OPCODE_LMULS_MF4
4365#undef CASE_VFMA_CHANGE_OPCODE_VV
4366#undef CASE_VFMA_CHANGE_OPCODE_SPLATS
4367
4368#undef CASE_RVV_OPCODE_UNMASK_LMUL
4369#undef CASE_RVV_OPCODE_MASK_LMUL
4370#undef CASE_RVV_OPCODE_LMUL
4371#undef CASE_RVV_OPCODE_UNMASK_WIDEN
4372#undef CASE_RVV_OPCODE_UNMASK
4373#undef CASE_RVV_OPCODE_MASK_WIDEN
4374#undef CASE_RVV_OPCODE_MASK
4375#undef CASE_RVV_OPCODE_WIDEN
4376#undef CASE_RVV_OPCODE
4377
4378#undef CASE_VMA_OPCODE_COMMON
4379#undef CASE_VMA_OPCODE_LMULS
4380#undef CASE_VFMA_OPCODE_COMMON
4381#undef CASE_VFMA_OPCODE_LMULS_M1
4382#undef CASE_VFMA_OPCODE_LMULS_MF2
4383#undef CASE_VFMA_OPCODE_LMULS_MF4
4384#undef CASE_VFMA_OPCODE_VV
4385#undef CASE_VFMA_SPLATS
4386
4387bool RISCVInstrInfo::simplifyInstruction(MachineInstr &MI) const {
4388 switch (MI.getOpcode()) {
4389 default:
4390 break;
4391 case RISCV::ADD:
4392 case RISCV::OR:
4393 case RISCV::XOR:
4394 // Normalize (so we hit the next if clause).
4395 // add/[x]or rd, zero, rs => add/[x]or rd, rs, zero
4396 if (MI.getOperand(i: 1).getReg() == RISCV::X0)
4397 commuteInstruction(MI);
4398 // add/[x]or rd, rs, zero => addi rd, rs, 0
4399 if (MI.getOperand(i: 2).getReg() == RISCV::X0) {
4400 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4401 MI.setDesc(get(Opcode: RISCV::ADDI));
4402 return true;
4403 }
4404 // xor rd, rs, rs => addi rd, zero, 0
4405 if (MI.getOpcode() == RISCV::XOR &&
4406 MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg()) {
4407 MI.getOperand(i: 1).setReg(RISCV::X0);
4408 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4409 MI.setDesc(get(Opcode: RISCV::ADDI));
4410 return true;
4411 }
4412 break;
4413 case RISCV::ORI:
4414 case RISCV::XORI:
4415 // [x]ori rd, zero, N => addi rd, zero, N
4416 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4417 MI.setDesc(get(Opcode: RISCV::ADDI));
4418 return true;
4419 }
4420 break;
4421 case RISCV::SUB:
4422 // sub rd, rs, zero => addi rd, rs, 0
4423 if (MI.getOperand(i: 2).getReg() == RISCV::X0) {
4424 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4425 MI.setDesc(get(Opcode: RISCV::ADDI));
4426 return true;
4427 }
4428 break;
4429 case RISCV::SUBW:
4430 // subw rd, rs, zero => addiw rd, rs, 0
4431 if (MI.getOperand(i: 2).getReg() == RISCV::X0) {
4432 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4433 MI.setDesc(get(Opcode: RISCV::ADDIW));
4434 return true;
4435 }
4436 break;
4437 case RISCV::ADDW:
4438 // Normalize (so we hit the next if clause).
4439 // addw rd, zero, rs => addw rd, rs, zero
4440 if (MI.getOperand(i: 1).getReg() == RISCV::X0)
4441 commuteInstruction(MI);
4442 // addw rd, rs, zero => addiw rd, rs, 0
4443 if (MI.getOperand(i: 2).getReg() == RISCV::X0) {
4444 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4445 MI.setDesc(get(Opcode: RISCV::ADDIW));
4446 return true;
4447 }
4448 break;
4449 case RISCV::SH1ADD:
4450 case RISCV::SH1ADD_UW:
4451 case RISCV::SH2ADD:
4452 case RISCV::SH2ADD_UW:
4453 case RISCV::SH3ADD:
4454 case RISCV::SH3ADD_UW:
4455 // shNadd[.uw] rd, zero, rs => addi rd, rs, 0
4456 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4457 MI.removeOperand(OpNo: 1);
4458 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
4459 MI.setDesc(get(Opcode: RISCV::ADDI));
4460 return true;
4461 }
4462 // shNadd[.uw] rd, rs, zero => slli[.uw] rd, rs, N
4463 if (MI.getOperand(i: 2).getReg() == RISCV::X0) {
4464 MI.removeOperand(OpNo: 2);
4465 unsigned Opc = MI.getOpcode();
4466 if (Opc == RISCV::SH1ADD_UW || Opc == RISCV::SH2ADD_UW ||
4467 Opc == RISCV::SH3ADD_UW) {
4468 MI.addOperand(Op: MachineOperand::CreateImm(Val: getSHXADDUWShiftAmount(Opc)));
4469 MI.setDesc(get(Opcode: RISCV::SLLI_UW));
4470 return true;
4471 }
4472 MI.addOperand(Op: MachineOperand::CreateImm(Val: getSHXADDShiftAmount(Opc)));
4473 MI.setDesc(get(Opcode: RISCV::SLLI));
4474 return true;
4475 }
4476 break;
4477 case RISCV::AND:
4478 case RISCV::MUL:
4479 case RISCV::MULH:
4480 case RISCV::MULHSU:
4481 case RISCV::MULHU:
4482 case RISCV::MULW:
4483 // and rd, zero, rs => addi rd, zero, 0
4484 // mul* rd, zero, rs => addi rd, zero, 0
4485 // and rd, rs, zero => addi rd, zero, 0
4486 // mul* rd, rs, zero => addi rd, zero, 0
4487 if (MI.getOperand(i: 1).getReg() == RISCV::X0 ||
4488 MI.getOperand(i: 2).getReg() == RISCV::X0) {
4489 MI.getOperand(i: 1).setReg(RISCV::X0);
4490 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4491 MI.setDesc(get(Opcode: RISCV::ADDI));
4492 return true;
4493 }
4494 break;
4495 case RISCV::ANDI:
4496 // andi rd, zero, C => addi rd, zero, 0
4497 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4498 MI.getOperand(i: 2).setImm(0);
4499 MI.setDesc(get(Opcode: RISCV::ADDI));
4500 return true;
4501 }
4502 break;
4503 case RISCV::SLL:
4504 case RISCV::SRL:
4505 case RISCV::SRA:
4506 // shift rd, zero, rs => addi rd, zero, 0
4507 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4508 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4509 MI.setDesc(get(Opcode: RISCV::ADDI));
4510 return true;
4511 }
4512 // shift rd, rs, zero => addi rd, rs, 0
4513 if (MI.getOperand(i: 2).getReg() == RISCV::X0) {
4514 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4515 MI.setDesc(get(Opcode: RISCV::ADDI));
4516 return true;
4517 }
4518 break;
4519 case RISCV::SLLW:
4520 case RISCV::SRLW:
4521 case RISCV::SRAW:
4522 // shiftw rd, zero, rs => addi rd, zero, 0
4523 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4524 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4525 MI.setDesc(get(Opcode: RISCV::ADDI));
4526 return true;
4527 }
4528 break;
4529 case RISCV::SLLI:
4530 case RISCV::SRLI:
4531 case RISCV::SRAI:
4532 case RISCV::SLLIW:
4533 case RISCV::SRLIW:
4534 case RISCV::SRAIW:
4535 case RISCV::SLLI_UW:
4536 // shiftimm rd, zero, N => addi rd, zero, 0
4537 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4538 MI.getOperand(i: 2).setImm(0);
4539 MI.setDesc(get(Opcode: RISCV::ADDI));
4540 return true;
4541 }
4542 break;
4543 case RISCV::SLTU:
4544 case RISCV::ADD_UW:
4545 // sltu rd, zero, zero => addi rd, zero, 0
4546 // add.uw rd, zero, zero => addi rd, zero, 0
4547 if (MI.getOperand(i: 1).getReg() == RISCV::X0 &&
4548 MI.getOperand(i: 2).getReg() == RISCV::X0) {
4549 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4550 MI.setDesc(get(Opcode: RISCV::ADDI));
4551 return true;
4552 }
4553 // add.uw rd, zero, rs => addi rd, rs, 0
4554 if (MI.getOpcode() == RISCV::ADD_UW &&
4555 MI.getOperand(i: 1).getReg() == RISCV::X0) {
4556 MI.removeOperand(OpNo: 1);
4557 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
4558 MI.setDesc(get(Opcode: RISCV::ADDI));
4559 }
4560 break;
4561 case RISCV::SLTIU:
4562 // sltiu rd, zero, NZC => addi rd, zero, 1
4563 // sltiu rd, zero, 0 => addi rd, zero, 0
4564 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4565 MI.getOperand(i: 2).setImm(MI.getOperand(i: 2).getImm() != 0);
4566 MI.setDesc(get(Opcode: RISCV::ADDI));
4567 return true;
4568 }
4569 break;
4570 case RISCV::SEXT_H:
4571 case RISCV::SEXT_B:
4572 case RISCV::ZEXT_H_RV32:
4573 case RISCV::ZEXT_H_RV64:
4574 // sext.[hb] rd, zero => addi rd, zero, 0
4575 // zext.h rd, zero => addi rd, zero, 0
4576 if (MI.getOperand(i: 1).getReg() == RISCV::X0) {
4577 MI.addOperand(Op: MachineOperand::CreateImm(Val: 0));
4578 MI.setDesc(get(Opcode: RISCV::ADDI));
4579 return true;
4580 }
4581 break;
4582 case RISCV::MIN:
4583 case RISCV::MINU:
4584 case RISCV::MAX:
4585 case RISCV::MAXU:
4586 // min|max rd, rs, rs => addi rd, rs, 0
4587 if (MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg()) {
4588 MI.getOperand(i: 2).ChangeToImmediate(ImmVal: 0);
4589 MI.setDesc(get(Opcode: RISCV::ADDI));
4590 return true;
4591 }
4592 break;
4593 case RISCV::BEQ:
4594 case RISCV::BNE:
4595 // b{eq,ne} zero, rs, imm => b{eq,ne} rs, zero, imm
4596 if (MI.getOperand(i: 0).getReg() == RISCV::X0) {
4597 MachineOperand MO0 = MI.getOperand(i: 0);
4598 MI.removeOperand(OpNo: 0);
4599 MI.insert(InsertBefore: MI.operands_begin() + 1, Ops: {MO0});
4600 }
4601 break;
4602 case RISCV::BLTU:
4603 // bltu zero, rs, imm => bne rs, zero, imm
4604 if (MI.getOperand(i: 0).getReg() == RISCV::X0) {
4605 MachineOperand MO0 = MI.getOperand(i: 0);
4606 MI.removeOperand(OpNo: 0);
4607 MI.insert(InsertBefore: MI.operands_begin() + 1, Ops: {MO0});
4608 MI.setDesc(get(Opcode: RISCV::BNE));
4609 }
4610 break;
4611 case RISCV::BGEU:
4612 // bgeu zero, rs, imm => beq rs, zero, imm
4613 if (MI.getOperand(i: 0).getReg() == RISCV::X0) {
4614 MachineOperand MO0 = MI.getOperand(i: 0);
4615 MI.removeOperand(OpNo: 0);
4616 MI.insert(InsertBefore: MI.operands_begin() + 1, Ops: {MO0});
4617 MI.setDesc(get(Opcode: RISCV::BEQ));
4618 }
4619 break;
4620 }
4621 return false;
4622}
4623
4624// clang-format off
4625#define CASE_WIDEOP_OPCODE_COMMON(OP, LMUL) \
4626 RISCV::PseudoV##OP##_##LMUL##_TIED
4627
4628#define CASE_WIDEOP_OPCODE_LMULS(OP) \
4629 CASE_WIDEOP_OPCODE_COMMON(OP, MF8): \
4630 case CASE_WIDEOP_OPCODE_COMMON(OP, MF4): \
4631 case CASE_WIDEOP_OPCODE_COMMON(OP, MF2): \
4632 case CASE_WIDEOP_OPCODE_COMMON(OP, M1): \
4633 case CASE_WIDEOP_OPCODE_COMMON(OP, M2): \
4634 case CASE_WIDEOP_OPCODE_COMMON(OP, M4)
4635
4636#define CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL) \
4637 case RISCV::PseudoV##OP##_##LMUL##_TIED: \
4638 NewOpc = RISCV::PseudoV##OP##_##LMUL; \
4639 break;
4640
4641#define CASE_WIDEOP_CHANGE_OPCODE_LMULS(OP) \
4642 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF8) \
4643 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4) \
4644 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2) \
4645 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1) \
4646 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2) \
4647 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4)
4648
4649// FP Widening Ops may by SEW aware. Create SEW aware cases for these cases.
4650#define CASE_FP_WIDEOP_OPCODE_COMMON(OP, LMUL, SEW) \
4651 RISCV::PseudoV##OP##_##LMUL##_##SEW##_TIED
4652
4653#define CASE_FP_WIDEOP_OPCODE_LMULS(OP) \
4654 CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF4, E16): \
4655 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E16): \
4656 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E32): \
4657 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E16): \
4658 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E32): \
4659 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E16): \
4660 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E32): \
4661 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E16): \
4662 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E32) \
4663
4664#define CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL, SEW) \
4665 case RISCV::PseudoV##OP##_##LMUL##_##SEW##_TIED: \
4666 NewOpc = RISCV::PseudoV##OP##_##LMUL##_##SEW; \
4667 break;
4668
4669#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(OP) \
4670 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4, E16) \
4671 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E16) \
4672 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E32) \
4673 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E16) \
4674 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E32) \
4675 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E16) \
4676 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E32) \
4677 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E16) \
4678 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E32) \
4679
4680#define CASE_FP_WIDEOP_OPCODE_LMULS_ALT(OP) \
4681 CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF4, E16): \
4682 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E16): \
4683 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E16): \
4684 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E16): \
4685 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E16)
4686
4687#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_ALT(OP) \
4688 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4, E16) \
4689 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E16) \
4690 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E16) \
4691 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E16) \
4692 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E16)
4693// clang-format on
4694
4695MachineInstr *RISCVInstrInfo::convertToThreeAddress(MachineInstr &MI,
4696 LiveVariables *LV,
4697 LiveIntervals *LIS) const {
4698 MachineInstrBuilder MIB;
4699 switch (MI.getOpcode()) {
4700 default:
4701 return nullptr;
4702 case CASE_FP_WIDEOP_OPCODE_LMULS_ALT(FWADD_ALT_WV):
4703 case CASE_FP_WIDEOP_OPCODE_LMULS_ALT(FWSUB_ALT_WV):
4704 case CASE_FP_WIDEOP_OPCODE_LMULS(FWADD_WV):
4705 case CASE_FP_WIDEOP_OPCODE_LMULS(FWSUB_WV): {
4706 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags) &&
4707 MI.getNumExplicitOperands() == 7 &&
4708 "Expect 7 explicit operands rd, rs2, rs1, rm, vl, sew, policy");
4709 // If the tail policy is undisturbed we can't convert.
4710 if ((MI.getOperand(i: RISCVII::getVecPolicyOpNum(Desc: MI.getDesc())).getImm() &
4711 1) == 0)
4712 return nullptr;
4713 // clang-format off
4714 unsigned NewOpc;
4715 switch (MI.getOpcode()) {
4716 default:
4717 llvm_unreachable("Unexpected opcode");
4718 CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(FWADD_WV)
4719 CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(FWSUB_WV)
4720 CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_ALT(FWADD_ALT_WV)
4721 CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_ALT(FWSUB_ALT_WV)
4722 }
4723 // clang-format on
4724
4725 MachineBasicBlock &MBB = *MI.getParent();
4726 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4727 .add(MO: MI.getOperand(i: 0))
4728 .addReg(RegNo: MI.getOperand(i: 0).getReg(), Flags: RegState::Undef)
4729 .add(MO: MI.getOperand(i: 1))
4730 .add(MO: MI.getOperand(i: 2))
4731 .add(MO: MI.getOperand(i: 3))
4732 .add(MO: MI.getOperand(i: 4))
4733 .add(MO: MI.getOperand(i: 5))
4734 .add(MO: MI.getOperand(i: 6));
4735 break;
4736 }
4737 case CASE_WIDEOP_OPCODE_LMULS(WADD_WV):
4738 case CASE_WIDEOP_OPCODE_LMULS(WADDU_WV):
4739 case CASE_WIDEOP_OPCODE_LMULS(WSUB_WV):
4740 case CASE_WIDEOP_OPCODE_LMULS(WSUBU_WV): {
4741 // If the tail policy is undisturbed we can't convert.
4742 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags) &&
4743 MI.getNumExplicitOperands() == 6);
4744 if ((MI.getOperand(i: RISCVII::getVecPolicyOpNum(Desc: MI.getDesc())).getImm() &
4745 1) == 0)
4746 return nullptr;
4747
4748 // clang-format off
4749 unsigned NewOpc;
4750 switch (MI.getOpcode()) {
4751 default:
4752 llvm_unreachable("Unexpected opcode");
4753 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WADD_WV)
4754 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WADDU_WV)
4755 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WSUB_WV)
4756 CASE_WIDEOP_CHANGE_OPCODE_LMULS(WSUBU_WV)
4757 }
4758 // clang-format on
4759
4760 MachineBasicBlock &MBB = *MI.getParent();
4761 MIB = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: get(Opcode: NewOpc))
4762 .add(MO: MI.getOperand(i: 0))
4763 .addReg(RegNo: MI.getOperand(i: 0).getReg(), Flags: RegState::Undef)
4764 .add(MO: MI.getOperand(i: 1))
4765 .add(MO: MI.getOperand(i: 2))
4766 .add(MO: MI.getOperand(i: 3))
4767 .add(MO: MI.getOperand(i: 4))
4768 .add(MO: MI.getOperand(i: 5));
4769 break;
4770 }
4771 }
4772 MIB.copyImplicitOps(OtherMI: MI);
4773
4774 if (LV) {
4775 unsigned NumOps = MI.getNumOperands();
4776 for (unsigned I = 1; I < NumOps; ++I) {
4777 MachineOperand &Op = MI.getOperand(i: I);
4778 if (Op.isReg() && Op.isKill())
4779 LV->replaceKillInstruction(Reg: Op.getReg(), OldMI&: MI, NewMI&: *MIB);
4780 }
4781 }
4782
4783 if (LIS) {
4784 SlotIndex Idx = LIS->ReplaceMachineInstrInMaps(MI, NewMI&: *MIB);
4785
4786 if (MI.getOperand(i: 0).isEarlyClobber()) {
4787 // Use operand 1 was tied to early-clobber def operand 0, so its live
4788 // interval could have ended at an early-clobber slot. Now they are not
4789 // tied we need to update it to the normal register slot.
4790 LiveInterval &LI = LIS->getInterval(Reg: MI.getOperand(i: 1).getReg());
4791 LiveRange::Segment *S = LI.getSegmentContaining(Idx);
4792 if (S->end == Idx.getRegSlot(EC: true))
4793 S->end = Idx.getRegSlot();
4794 }
4795 }
4796
4797 return MIB;
4798}
4799
4800#undef CASE_WIDEOP_OPCODE_COMMON
4801#undef CASE_WIDEOP_OPCODE_LMULS
4802#undef CASE_WIDEOP_CHANGE_OPCODE_COMMON
4803#undef CASE_WIDEOP_CHANGE_OPCODE_LMULS
4804#undef CASE_FP_WIDEOP_OPCODE_COMMON
4805#undef CASE_FP_WIDEOP_OPCODE_LMULS
4806#undef CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON
4807#undef CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS
4808
4809void RISCVInstrInfo::mulImm(MachineFunction &MF, MachineBasicBlock &MBB,
4810 MachineBasicBlock::iterator II, const DebugLoc &DL,
4811 Register DestReg, uint32_t Amount,
4812 MachineInstr::MIFlag Flag) const {
4813 MachineRegisterInfo &MRI = MF.getRegInfo();
4814 if (llvm::has_single_bit<uint32_t>(Value: Amount)) {
4815 uint32_t ShiftAmount = Log2_32(Value: Amount);
4816 if (ShiftAmount == 0)
4817 return;
4818 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::SLLI), DestReg)
4819 .addReg(RegNo: DestReg, Flags: RegState::Kill)
4820 .addImm(Val: ShiftAmount)
4821 .setMIFlag(Flag);
4822 } else if (int ShXAmount, ShiftAmount;
4823 STI.hasShlAdd(ShAmt: 3) &&
4824 (ShXAmount = isShifted359(Value: Amount, Shift&: ShiftAmount)) != 0) {
4825 // We can use Zba SHXADD+SLLI instructions for multiply in some cases.
4826 unsigned Opc;
4827 switch (ShXAmount) {
4828 case 1:
4829 Opc = RISCV::SH1ADD;
4830 break;
4831 case 2:
4832 Opc = RISCV::SH2ADD;
4833 break;
4834 case 3:
4835 Opc = RISCV::SH3ADD;
4836 break;
4837 default:
4838 llvm_unreachable("unexpected result of isShifted359");
4839 }
4840 if (ShiftAmount)
4841 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::SLLI), DestReg)
4842 .addReg(RegNo: DestReg, Flags: RegState::Kill)
4843 .addImm(Val: ShiftAmount)
4844 .setMIFlag(Flag);
4845 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: Opc), DestReg)
4846 .addReg(RegNo: DestReg, Flags: RegState::Kill)
4847 .addReg(RegNo: DestReg)
4848 .setMIFlag(Flag);
4849 } else if (llvm::has_single_bit<uint32_t>(Value: Amount - 1)) {
4850 Register ScaledRegister = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
4851 uint32_t ShiftAmount = Log2_32(Value: Amount - 1);
4852 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::SLLI), DestReg: ScaledRegister)
4853 .addReg(RegNo: DestReg)
4854 .addImm(Val: ShiftAmount)
4855 .setMIFlag(Flag);
4856 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::ADD), DestReg)
4857 .addReg(RegNo: ScaledRegister, Flags: RegState::Kill)
4858 .addReg(RegNo: DestReg, Flags: RegState::Kill)
4859 .setMIFlag(Flag);
4860 } else if (llvm::has_single_bit<uint32_t>(Value: Amount + 1)) {
4861 Register ScaledRegister = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
4862 uint32_t ShiftAmount = Log2_32(Value: Amount + 1);
4863 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::SLLI), DestReg: ScaledRegister)
4864 .addReg(RegNo: DestReg)
4865 .addImm(Val: ShiftAmount)
4866 .setMIFlag(Flag);
4867 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::SUB), DestReg)
4868 .addReg(RegNo: ScaledRegister, Flags: RegState::Kill)
4869 .addReg(RegNo: DestReg, Flags: RegState::Kill)
4870 .setMIFlag(Flag);
4871 } else if (STI.hasStdExtZmmul()) {
4872 Register N = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
4873 movImm(MBB, MBBI: II, DL, DstReg: N, Val: Amount, Flag);
4874 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::MUL), DestReg)
4875 .addReg(RegNo: DestReg, Flags: RegState::Kill)
4876 .addReg(RegNo: N, Flags: RegState::Kill)
4877 .setMIFlag(Flag);
4878 } else {
4879 Register Acc;
4880 uint32_t PrevShiftAmount = 0;
4881 for (uint32_t ShiftAmount = 0; Amount >> ShiftAmount; ShiftAmount++) {
4882 if (Amount & (1U << ShiftAmount)) {
4883 if (ShiftAmount)
4884 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::SLLI), DestReg)
4885 .addReg(RegNo: DestReg, Flags: RegState::Kill)
4886 .addImm(Val: ShiftAmount - PrevShiftAmount)
4887 .setMIFlag(Flag);
4888 if (Amount >> (ShiftAmount + 1)) {
4889 // If we don't have an accmulator yet, create it and copy DestReg.
4890 if (!Acc) {
4891 Acc = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
4892 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: TargetOpcode::COPY), DestReg: Acc)
4893 .addReg(RegNo: DestReg)
4894 .setMIFlag(Flag);
4895 } else {
4896 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::ADD), DestReg: Acc)
4897 .addReg(RegNo: Acc, Flags: RegState::Kill)
4898 .addReg(RegNo: DestReg)
4899 .setMIFlag(Flag);
4900 }
4901 }
4902 PrevShiftAmount = ShiftAmount;
4903 }
4904 }
4905 assert(Acc && "Expected valid accumulator");
4906 BuildMI(BB&: MBB, I: II, MIMD: DL, MCID: get(Opcode: RISCV::ADD), DestReg)
4907 .addReg(RegNo: DestReg, Flags: RegState::Kill)
4908 .addReg(RegNo: Acc, Flags: RegState::Kill)
4909 .setMIFlag(Flag);
4910 }
4911}
4912
4913ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
4914RISCVInstrInfo::getSerializableMachineMemOperandTargetFlags() const {
4915 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
4916 {{MONontemporalBit0, "riscv-nontemporal-domain-bit-0"},
4917 {MONontemporalBit1, "riscv-nontemporal-domain-bit-1"}};
4918 return ArrayRef(TargetFlags);
4919}
4920
4921unsigned RISCVInstrInfo::getTailDuplicateSize(CodeGenOptLevel OptLevel) const {
4922 return OptLevel >= CodeGenOptLevel::Aggressive
4923 ? STI.getTailDupAggressiveThreshold()
4924 : 2;
4925}
4926
4927bool RISCV::isRVVSpill(const MachineInstr &MI) {
4928 // RVV lacks any support for immediate addressing for stack addresses, so be
4929 // conservative.
4930 unsigned Opcode = MI.getOpcode();
4931 if (!RISCVVPseudosTable::getPseudoInfo(Pseudo: Opcode) &&
4932 !getLMULForRVVWholeLoadStore(Opcode) && !isRVVSpillForZvlsseg(Opcode))
4933 return false;
4934 return true;
4935}
4936
4937/// Return true if \p MI is a copy that will be lowered to one or more vmvNr.vs.
4938bool RISCV::isVectorCopy(const TargetRegisterInfo *TRI,
4939 const MachineInstr &MI) {
4940 return MI.isCopy() && MI.getOperand(i: 0).getReg().isPhysical() &&
4941 RISCVRegisterInfo::isRVVRegClass(
4942 RC: TRI->getMinimalPhysRegClass(Reg: MI.getOperand(i: 0).getReg()));
4943}
4944
4945std::optional<std::pair<unsigned, unsigned>>
4946RISCV::isRVVSpillForZvlsseg(unsigned Opcode) {
4947 switch (Opcode) {
4948 default:
4949 return std::nullopt;
4950 case RISCV::PseudoVSPILL2_M1:
4951 case RISCV::PseudoVRELOAD2_M1:
4952 return std::make_pair(x: 2u, y: 1u);
4953 case RISCV::PseudoVSPILL2_M2:
4954 case RISCV::PseudoVRELOAD2_M2:
4955 return std::make_pair(x: 2u, y: 2u);
4956 case RISCV::PseudoVSPILL2_M4:
4957 case RISCV::PseudoVRELOAD2_M4:
4958 return std::make_pair(x: 2u, y: 4u);
4959 case RISCV::PseudoVSPILL3_M1:
4960 case RISCV::PseudoVRELOAD3_M1:
4961 return std::make_pair(x: 3u, y: 1u);
4962 case RISCV::PseudoVSPILL3_M2:
4963 case RISCV::PseudoVRELOAD3_M2:
4964 return std::make_pair(x: 3u, y: 2u);
4965 case RISCV::PseudoVSPILL4_M1:
4966 case RISCV::PseudoVRELOAD4_M1:
4967 return std::make_pair(x: 4u, y: 1u);
4968 case RISCV::PseudoVSPILL4_M2:
4969 case RISCV::PseudoVRELOAD4_M2:
4970 return std::make_pair(x: 4u, y: 2u);
4971 case RISCV::PseudoVSPILL5_M1:
4972 case RISCV::PseudoVRELOAD5_M1:
4973 return std::make_pair(x: 5u, y: 1u);
4974 case RISCV::PseudoVSPILL6_M1:
4975 case RISCV::PseudoVRELOAD6_M1:
4976 return std::make_pair(x: 6u, y: 1u);
4977 case RISCV::PseudoVSPILL7_M1:
4978 case RISCV::PseudoVRELOAD7_M1:
4979 return std::make_pair(x: 7u, y: 1u);
4980 case RISCV::PseudoVSPILL8_M1:
4981 case RISCV::PseudoVRELOAD8_M1:
4982 return std::make_pair(x: 8u, y: 1u);
4983 }
4984}
4985
4986bool RISCV::hasEqualFRM(const MachineInstr &MI1, const MachineInstr &MI2) {
4987 int16_t MI1FrmOpIdx =
4988 RISCV::getNamedOperandIdx(Opcode: MI1.getOpcode(), Name: RISCV::OpName::frm);
4989 int16_t MI2FrmOpIdx =
4990 RISCV::getNamedOperandIdx(Opcode: MI2.getOpcode(), Name: RISCV::OpName::frm);
4991 if (MI1FrmOpIdx < 0 || MI2FrmOpIdx < 0)
4992 return false;
4993 MachineOperand FrmOp1 = MI1.getOperand(i: MI1FrmOpIdx);
4994 MachineOperand FrmOp2 = MI2.getOperand(i: MI2FrmOpIdx);
4995 return FrmOp1.getImm() == FrmOp2.getImm();
4996}
4997
4998std::optional<unsigned>
4999RISCV::getVectorLowDemandedScalarBits(unsigned Opcode, unsigned Log2SEW) {
5000 switch (Opcode) {
5001 default:
5002 return std::nullopt;
5003
5004 // 11.6. Vector Single-Width Shift Instructions
5005 case RISCV::VSLL_VX:
5006 case RISCV::VSRL_VX:
5007 case RISCV::VSRA_VX:
5008 // 12.4. Vector Single-Width Scaling Shift Instructions
5009 case RISCV::VSSRL_VX:
5010 case RISCV::VSSRA_VX:
5011 // Zvbb
5012 case RISCV::VROL_VX:
5013 case RISCV::VROR_VX:
5014 // Only the low lg2(SEW) bits of the shift-amount value are used.
5015 return Log2SEW;
5016
5017 // 11.7 Vector Narrowing Integer Right Shift Instructions
5018 case RISCV::VNSRL_WX:
5019 case RISCV::VNSRA_WX:
5020 // 12.5. Vector Narrowing Fixed-Point Clip Instructions
5021 case RISCV::VNCLIPU_WX:
5022 case RISCV::VNCLIP_WX:
5023 // Zvbb
5024 case RISCV::VWSLL_VX:
5025 // Only the low lg2(2*SEW) bits of the shift-amount value are used.
5026 return Log2SEW + 1;
5027
5028 // 11.1. Vector Single-Width Integer Add and Subtract
5029 case RISCV::VADD_VX:
5030 case RISCV::VSUB_VX:
5031 case RISCV::VRSUB_VX:
5032 // 11.2. Vector Widening Integer Add/Subtract
5033 case RISCV::VWADDU_VX:
5034 case RISCV::VWSUBU_VX:
5035 case RISCV::VWADD_VX:
5036 case RISCV::VWSUB_VX:
5037 case RISCV::VWADDU_WX:
5038 case RISCV::VWSUBU_WX:
5039 case RISCV::VWADD_WX:
5040 case RISCV::VWSUB_WX:
5041 // 11.4. Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions
5042 case RISCV::VADC_VXM:
5043 case RISCV::VADC_VIM:
5044 case RISCV::VMADC_VXM:
5045 case RISCV::VMADC_VIM:
5046 case RISCV::VMADC_VX:
5047 case RISCV::VSBC_VXM:
5048 case RISCV::VMSBC_VXM:
5049 case RISCV::VMSBC_VX:
5050 // 11.5 Vector Bitwise Logical Instructions
5051 case RISCV::VAND_VX:
5052 case RISCV::VOR_VX:
5053 case RISCV::VXOR_VX:
5054 // 11.8. Vector Integer Compare Instructions
5055 case RISCV::VMSEQ_VX:
5056 case RISCV::VMSNE_VX:
5057 case RISCV::VMSLTU_VX:
5058 case RISCV::VMSLT_VX:
5059 case RISCV::VMSLEU_VX:
5060 case RISCV::VMSLE_VX:
5061 case RISCV::VMSGTU_VX:
5062 case RISCV::VMSGT_VX:
5063 // 11.9. Vector Integer Min/Max Instructions
5064 case RISCV::VMINU_VX:
5065 case RISCV::VMIN_VX:
5066 case RISCV::VMAXU_VX:
5067 case RISCV::VMAX_VX:
5068 // 11.10. Vector Single-Width Integer Multiply Instructions
5069 case RISCV::VMUL_VX:
5070 case RISCV::VMULH_VX:
5071 case RISCV::VMULHU_VX:
5072 case RISCV::VMULHSU_VX:
5073 // 11.11. Vector Integer Divide Instructions
5074 case RISCV::VDIVU_VX:
5075 case RISCV::VDIV_VX:
5076 case RISCV::VREMU_VX:
5077 case RISCV::VREM_VX:
5078 // 11.12. Vector Widening Integer Multiply Instructions
5079 case RISCV::VWMUL_VX:
5080 case RISCV::VWMULU_VX:
5081 case RISCV::VWMULSU_VX:
5082 // 11.13. Vector Single-Width Integer Multiply-Add Instructions
5083 case RISCV::VMACC_VX:
5084 case RISCV::VNMSAC_VX:
5085 case RISCV::VMADD_VX:
5086 case RISCV::VNMSUB_VX:
5087 // 11.14. Vector Widening Integer Multiply-Add Instructions
5088 case RISCV::VWMACCU_VX:
5089 case RISCV::VWMACC_VX:
5090 case RISCV::VWMACCSU_VX:
5091 case RISCV::VWMACCUS_VX:
5092 // 11.15. Vector Integer Merge Instructions
5093 case RISCV::VMERGE_VXM:
5094 // 11.16. Vector Integer Move Instructions
5095 case RISCV::VMV_V_X:
5096 // 12.1. Vector Single-Width Saturating Add and Subtract
5097 case RISCV::VSADDU_VX:
5098 case RISCV::VSADD_VX:
5099 case RISCV::VSSUBU_VX:
5100 case RISCV::VSSUB_VX:
5101 // 12.2. Vector Single-Width Averaging Add and Subtract
5102 case RISCV::VAADDU_VX:
5103 case RISCV::VAADD_VX:
5104 case RISCV::VASUBU_VX:
5105 case RISCV::VASUB_VX:
5106 // 12.3. Vector Single-Width Fractional Multiply with Rounding and Saturation
5107 case RISCV::VSMUL_VX:
5108 // 16.1. Integer Scalar Move Instructions
5109 case RISCV::VMV_S_X:
5110 // Zvbb
5111 case RISCV::VANDN_VX:
5112 return 1U << Log2SEW;
5113 }
5114}
5115
5116unsigned RISCV::getRVVMCOpcode(unsigned RVVPseudoOpcode) {
5117 const RISCVVPseudosTable::PseudoInfo *RVV =
5118 RISCVVPseudosTable::getPseudoInfo(Pseudo: RVVPseudoOpcode);
5119 if (!RVV)
5120 return 0;
5121 return RVV->BaseInstr;
5122}
5123
5124unsigned RISCV::getDestLog2EEW(const MCInstrDesc &Desc, unsigned Log2SEW) {
5125 unsigned DestEEW =
5126 (Desc.TSFlags & RISCVII::DestEEWMask) >> RISCVII::DestEEWShift;
5127 // EEW = 1
5128 if (DestEEW == 0)
5129 return 0;
5130 // EEW = SEW * n
5131 unsigned Scaled = Log2SEW + (DestEEW - 1);
5132 assert(Scaled >= 3 && Scaled <= 6);
5133 return Scaled;
5134}
5135
5136static std::optional<int64_t> getEffectiveImm(const MachineOperand &MO) {
5137 assert(MO.isImm() || MO.getReg().isVirtual());
5138 if (MO.isImm())
5139 return MO.getImm();
5140 const MachineInstr *Def =
5141 MO.getParent()->getMF()->getRegInfo().getVRegDef(Reg: MO.getReg());
5142 int64_t Imm;
5143 if (isLoadImm(MI: Def, Imm))
5144 return Imm;
5145 return std::nullopt;
5146}
5147
5148/// Given two VL operands, do we know that LHS <= RHS? Must be used in SSA form.
5149bool RISCV::isVLKnownLE(const MachineOperand &LHS, const MachineOperand &RHS) {
5150 assert((LHS.isImm() || LHS.getParent()->getMF()->getRegInfo().isSSA()) &&
5151 (RHS.isImm() || RHS.getParent()->getMF()->getRegInfo().isSSA()));
5152 if (LHS.isReg() && RHS.isReg() && LHS.getReg().isVirtual() &&
5153 LHS.getReg() == RHS.getReg())
5154 return true;
5155 if (RHS.isImm() && RHS.getImm() == RISCV::VLMaxSentinel)
5156 return true;
5157 if (LHS.isImm() && LHS.getImm() == 0)
5158 return true;
5159 if (LHS.isImm() && LHS.getImm() == RISCV::VLMaxSentinel)
5160 return false;
5161 std::optional<int64_t> LHSImm = getEffectiveImm(MO: LHS),
5162 RHSImm = getEffectiveImm(MO: RHS);
5163 if (!LHSImm || !RHSImm)
5164 return false;
5165 return LHSImm <= RHSImm;
5166}
5167
5168namespace {
5169class RISCVPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
5170 const MachineInstr *LHS;
5171 const MachineInstr *RHS;
5172 SmallVector<MachineOperand, 3> Cond;
5173
5174public:
5175 RISCVPipelinerLoopInfo(const MachineInstr *LHS, const MachineInstr *RHS,
5176 const SmallVectorImpl<MachineOperand> &Cond)
5177 : LHS(LHS), RHS(RHS), Cond(Cond.begin(), Cond.end()) {}
5178
5179 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
5180 // Make the instructions for loop control be placed in stage 0.
5181 // The predecessors of LHS/RHS are considered by the caller.
5182 if (LHS && MI == LHS)
5183 return true;
5184 if (RHS && MI == RHS)
5185 return true;
5186 return false;
5187 }
5188
5189 std::optional<bool> createTripCountGreaterCondition(
5190 int TC, MachineBasicBlock &MBB,
5191 SmallVectorImpl<MachineOperand> &CondParam) override {
5192 // A branch instruction will be inserted as "if (Cond) goto epilogue".
5193 // Cond is normalized for such use.
5194 // The predecessors of the branch are assumed to have already been inserted.
5195 CondParam = Cond;
5196 return {};
5197 }
5198
5199 void setPreheader(MachineBasicBlock *NewPreheader) override {}
5200
5201 void adjustTripCount(int TripCountAdjust) override {}
5202};
5203} // namespace
5204
5205std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
5206RISCVInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
5207 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
5208 SmallVector<MachineOperand, 4> Cond;
5209 if (analyzeBranch(MBB&: *LoopBB, TBB, FBB, Cond, /*AllowModify=*/false))
5210 return nullptr;
5211
5212 // Infinite loops are not supported
5213 if (TBB == LoopBB && FBB == LoopBB)
5214 return nullptr;
5215
5216 // Must be conditional branch
5217 if (FBB == nullptr)
5218 return nullptr;
5219
5220 assert((TBB == LoopBB || FBB == LoopBB) &&
5221 "The Loop must be a single-basic-block loop");
5222
5223 // Normalization for createTripCountGreaterCondition()
5224 if (TBB == LoopBB)
5225 reverseBranchCondition(Cond);
5226
5227 const MachineRegisterInfo &MRI = LoopBB->getParent()->getRegInfo();
5228 auto FindRegDef = [&MRI](MachineOperand &Op) -> const MachineInstr * {
5229 if (!Op.isReg())
5230 return nullptr;
5231 Register Reg = Op.getReg();
5232 if (!Reg.isVirtual())
5233 return nullptr;
5234 return MRI.getVRegDef(Reg);
5235 };
5236
5237 const MachineInstr *LHS = FindRegDef(Cond[1]);
5238 const MachineInstr *RHS = FindRegDef(Cond[2]);
5239 if (LHS && LHS->isPHI())
5240 return nullptr;
5241 if (RHS && RHS->isPHI())
5242 return nullptr;
5243
5244 return std::make_unique<RISCVPipelinerLoopInfo>(args&: LHS, args&: RHS, args&: Cond);
5245}
5246
5247// FIXME: We should remove this if we have a default generic scheduling model.
5248bool RISCVInstrInfo::isHighLatencyDef(int Opc) const {
5249 unsigned RVVMCOpcode = RISCV::getRVVMCOpcode(RVVPseudoOpcode: Opc);
5250 Opc = RVVMCOpcode ? RVVMCOpcode : Opc;
5251 switch (Opc) {
5252 default:
5253 return false;
5254 // Integer div/rem.
5255 case RISCV::DIV:
5256 case RISCV::DIVW:
5257 case RISCV::DIVU:
5258 case RISCV::DIVUW:
5259 case RISCV::REM:
5260 case RISCV::REMW:
5261 case RISCV::REMU:
5262 case RISCV::REMUW:
5263 // Floating-point div/sqrt.
5264 case RISCV::FDIV_H:
5265 case RISCV::FDIV_S:
5266 case RISCV::FDIV_D:
5267 case RISCV::FDIV_H_INX:
5268 case RISCV::FDIV_S_INX:
5269 case RISCV::FDIV_D_INX:
5270 case RISCV::FDIV_D_IN32X:
5271 case RISCV::FSQRT_H:
5272 case RISCV::FSQRT_S:
5273 case RISCV::FSQRT_D:
5274 case RISCV::FSQRT_H_INX:
5275 case RISCV::FSQRT_S_INX:
5276 case RISCV::FSQRT_D_INX:
5277 case RISCV::FSQRT_D_IN32X:
5278 // Vector integer div/rem
5279 case RISCV::VDIV_VV:
5280 case RISCV::VDIV_VX:
5281 case RISCV::VDIVU_VV:
5282 case RISCV::VDIVU_VX:
5283 case RISCV::VREM_VV:
5284 case RISCV::VREM_VX:
5285 case RISCV::VREMU_VV:
5286 case RISCV::VREMU_VX:
5287 // Vector floating-point div/sqrt.
5288 case RISCV::VFDIV_VV:
5289 case RISCV::VFDIV_VF:
5290 case RISCV::VFRDIV_VF:
5291 case RISCV::VFSQRT_V:
5292 case RISCV::VFRSQRT7_V:
5293 return true;
5294 }
5295}
5296
5297bool RISCVInstrInfo::isVRegCopy(const MachineInstr *MI, unsigned LMul) const {
5298 if (MI->getOpcode() != TargetOpcode::COPY)
5299 return false;
5300 const MachineRegisterInfo &MRI = MI->getMF()->getRegInfo();
5301 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
5302
5303 Register DstReg = MI->getOperand(i: 0).getReg();
5304 const TargetRegisterClass *RC = DstReg.isVirtual()
5305 ? MRI.getRegClass(Reg: DstReg)
5306 : TRI->getMinimalPhysRegClass(Reg: DstReg);
5307
5308 if (!RISCVRegisterInfo::isRVVRegClass(RC))
5309 return false;
5310
5311 if (!LMul)
5312 return true;
5313
5314 // TODO: Perhaps we could distinguish segment register classes (e.g. VRN3M2)
5315 // in the future.
5316 auto [RCLMul, RCFractional] =
5317 RISCVVType::decodeVLMUL(VLMul: RISCVRI::getLMul(TSFlags: RC->TSFlags));
5318 return (!RCFractional && LMul == RCLMul) || (RCFractional && LMul == 1);
5319}
5320