1//===-- llvm/CodeGen/GlobalISel/MachineIRBuilder.cpp - MIBuilder--*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the MachineIRBuidler class.
10//===----------------------------------------------------------------------===//
11#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
12#include "llvm/CodeGen/Analysis.h"
13#include "llvm/CodeGen/MachineFunction.h"
14#include "llvm/CodeGen/MachineInstr.h"
15#include "llvm/CodeGen/MachineInstrBuilder.h"
16#include "llvm/CodeGen/MachineRegisterInfo.h"
17#include "llvm/CodeGen/TargetInstrInfo.h"
18#include "llvm/CodeGen/TargetLowering.h"
19#include "llvm/CodeGen/TargetOpcodes.h"
20#include "llvm/CodeGen/TargetSubtargetInfo.h"
21#include "llvm/IR/DebugInfoMetadata.h"
22
23using namespace llvm;
24
25void MachineIRBuilder::setMF(MachineFunction &MF) {
26 State.MF = &MF;
27 State.MBB = nullptr;
28 State.MRI = &MF.getRegInfo();
29 State.TII = MF.getSubtarget().getInstrInfo();
30 State.DL = DebugLoc();
31 State.PCSections = nullptr;
32 State.MMRA = nullptr;
33 State.II = MachineBasicBlock::iterator();
34 State.Observer = nullptr;
35}
36
37//------------------------------------------------------------------------------
38// Build instruction variants.
39//------------------------------------------------------------------------------
40
41MachineInstrBuilder MachineIRBuilder::buildInstrNoInsert(unsigned Opcode) {
42 return BuildMI(
43 MF&: getMF(),
44 MIMD: {getDL(), getPCSections(), getMMRAMetadata(), getDeactivationSymbol()},
45 MCID: getTII().get(Opcode));
46}
47
48MachineInstrBuilder MachineIRBuilder::insertInstr(MachineInstrBuilder MIB) {
49 getMBB().insert(I: getInsertPt(), MI: MIB);
50 recordInsertion(InsertedInstr: MIB);
51 return MIB;
52}
53
54MachineInstrBuilder
55MachineIRBuilder::buildDirectDbgValue(Register Reg, const MDNode *Variable,
56 const MDNode *Expr) {
57 assert(isa<DILocalVariable>(Variable) && "not a variable");
58 assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
59 assert(
60 cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
61 "Expected inlined-at fields to agree");
62 return insertInstr(MIB: BuildMI(MF&: getMF(), DL: getDL(),
63 MCID: getTII().get(Opcode: TargetOpcode::DBG_VALUE),
64 /*IsIndirect*/ false, Reg, Variable, Expr));
65}
66
67MachineInstrBuilder
68MachineIRBuilder::buildIndirectDbgValue(Register Reg, const MDNode *Variable,
69 const MDNode *Expr) {
70 assert(isa<DILocalVariable>(Variable) && "not a variable");
71 assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
72 assert(
73 cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
74 "Expected inlined-at fields to agree");
75 return insertInstr(MIB: BuildMI(MF&: getMF(), DL: getDL(),
76 MCID: getTII().get(Opcode: TargetOpcode::DBG_VALUE),
77 /*IsIndirect*/ true, Reg, Variable, Expr));
78}
79
80MachineInstrBuilder MachineIRBuilder::buildFIDbgValue(int FI,
81 const MDNode *Variable,
82 const MDNode *Expr) {
83 assert(isa<DILocalVariable>(Variable) && "not a variable");
84 assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
85 assert(
86 cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
87 "Expected inlined-at fields to agree");
88 return insertInstr(MIB: buildInstrNoInsert(Opcode: TargetOpcode::DBG_VALUE)
89 .addFrameIndex(Idx: FI)
90 .addImm(Val: 0)
91 .addMetadata(MD: Variable)
92 .addMetadata(MD: Expr));
93}
94
95MachineInstrBuilder MachineIRBuilder::buildConstDbgValue(const Constant &C,
96 const MDNode *Variable,
97 const MDNode *Expr) {
98 assert(isa<DILocalVariable>(Variable) && "not a variable");
99 assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
100 assert(
101 cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
102 "Expected inlined-at fields to agree");
103 auto MIB = buildInstrNoInsert(Opcode: TargetOpcode::DBG_VALUE);
104
105 auto *NumericConstant = [&] () -> const Constant* {
106 if (const auto *CE = dyn_cast<ConstantExpr>(Val: &C))
107 if (CE->getOpcode() == Instruction::IntToPtr)
108 return CE->getOperand(i_nocapture: 0);
109 return &C;
110 }();
111
112 bool IsIndirect = true;
113 int64_t GlobalOffset;
114 if (auto *CI = dyn_cast<ConstantInt>(Val: NumericConstant)) {
115 if (CI->getBitWidth() > 64)
116 MIB.addCImm(Val: CI);
117 else if (CI->getBitWidth() == 1)
118 MIB.addImm(Val: CI->getZExtValue());
119 else
120 MIB.addImm(Val: CI->getSExtValue());
121 } else if (auto *CFP = dyn_cast<ConstantFP>(Val: NumericConstant)) {
122 MIB.addFPImm(Val: CFP);
123 } else if (isa<ConstantPointerNull>(Val: NumericConstant)) {
124 MIB.addImm(Val: 0);
125 } else if (const GlobalValue *GV = getDescribableGlobalAddress(
126 C: NumericConstant, Offset&: GlobalOffset, MF: getMF())) {
127 // The address of a global is a direct link-time constant. A displacement
128 // from it rides along in the expression rather than in the operand.
129 MIB.addGlobalAddress(GV);
130 if (GlobalOffset) {
131 SmallVector<uint64_t, 3> Ops;
132 DIExpression::appendOffset(Ops, Offset: GlobalOffset);
133 Expr = DIExpression::appendOpsToArg(Expr: cast<DIExpression>(Val: Expr), Ops, ArgNo: 0,
134 /*StackValue=*/false);
135 }
136 IsIndirect = false;
137 } else {
138 // Insert $noreg if we didn't find a usable constant and had to drop it.
139 MIB.addReg(RegNo: Register());
140 }
141
142 // DBG_VALUE spells an indirect location with a zero immediate offset operand
143 // and a direct one with $noreg. isIndirectDebugValue() ignores the offset for
144 // a non-register location operand, but isDebugOffsetImm() does not, and
145 // several consumers ask that instead.
146 if (IsIndirect)
147 MIB.addImm(Val: 0);
148 else
149 MIB.addReg(RegNo: Register());
150 MIB.addMetadata(MD: Variable).addMetadata(MD: Expr);
151 return insertInstr(MIB);
152}
153
154MachineInstrBuilder MachineIRBuilder::buildDbgLabel(const MDNode *Label) {
155 assert(isa<DILabel>(Label) && "not a label");
156 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(State.DL) &&
157 "Expected inlined-at fields to agree");
158 auto MIB = buildInstr(Opcode: TargetOpcode::DBG_LABEL);
159
160 return MIB.addMetadata(MD: Label);
161}
162
163MachineInstrBuilder MachineIRBuilder::buildDynStackAlloc(const DstOp &Res,
164 const SrcOp &Size,
165 Align Alignment) {
166 assert(Res.getLLTTy(*getMRI()).isPointer() && "expected ptr dst type");
167 auto MIB = buildInstr(Opcode: TargetOpcode::G_DYN_STACKALLOC);
168 Res.addDefToMIB(MRI&: *getMRI(), MIB);
169 Size.addSrcToMIB(MIB);
170 MIB.addImm(Val: Alignment.value());
171 return MIB;
172}
173
174MachineInstrBuilder MachineIRBuilder::buildFrameIndex(const DstOp &Res,
175 int Idx) {
176 assert(Res.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
177 auto MIB = buildInstr(Opcode: TargetOpcode::G_FRAME_INDEX);
178 Res.addDefToMIB(MRI&: *getMRI(), MIB);
179 MIB.addFrameIndex(Idx);
180 return MIB;
181}
182
183MachineInstrBuilder MachineIRBuilder::buildGlobalValue(const DstOp &Res,
184 const GlobalValue *GV) {
185 assert(Res.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
186 assert(Res.getLLTTy(*getMRI()).getAddressSpace() ==
187 GV->getType()->getAddressSpace() &&
188 "address space mismatch");
189
190 auto MIB = buildInstr(Opcode: TargetOpcode::G_GLOBAL_VALUE);
191 Res.addDefToMIB(MRI&: *getMRI(), MIB);
192 MIB.addGlobalAddress(GV);
193 return MIB;
194}
195
196MachineInstrBuilder MachineIRBuilder::buildConstantPool(const DstOp &Res,
197 unsigned Idx) {
198 assert(Res.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
199 auto MIB = buildInstr(Opcode: TargetOpcode::G_CONSTANT_POOL);
200 Res.addDefToMIB(MRI&: *getMRI(), MIB);
201 MIB.addConstantPoolIndex(Idx);
202 return MIB;
203}
204
205MachineInstrBuilder MachineIRBuilder::buildJumpTable(const LLT PtrTy,
206 unsigned JTI) {
207 return buildInstr(Opc: TargetOpcode::G_JUMP_TABLE, DstOps: {PtrTy}, SrcOps: {})
208 .addJumpTableIndex(Idx: JTI);
209}
210
211void MachineIRBuilder::validateUnaryOp(const LLT Res, const LLT Op0) {
212 assert((Res.isScalar() || Res.isVector()) && "invalid operand type");
213 assert((Res == Op0) && "type mismatch");
214}
215
216void MachineIRBuilder::validateBinaryOp(const LLT Res, const LLT Op0,
217 const LLT Op1) {
218 assert((Res.isScalar() || Res.isVector()) && "invalid operand type");
219 assert((Res == Op0 && Res == Op1) && "type mismatch");
220}
221
222void MachineIRBuilder::validateShiftOp(const LLT Res, const LLT Op0,
223 const LLT Op1) {
224 assert((Res.isScalar() || Res.isVector()) && "invalid operand type");
225 assert((Res == Op0) && "type mismatch");
226}
227
228MachineInstrBuilder
229MachineIRBuilder::buildPtrAdd(const DstOp &Res, const SrcOp &Op0,
230 const SrcOp &Op1, std::optional<unsigned> Flags) {
231 assert(Res.getLLTTy(*getMRI()).isPointerOrPointerVector() &&
232 Res.getLLTTy(*getMRI()) == Op0.getLLTTy(*getMRI()) && "type mismatch");
233 assert(Op1.getLLTTy(*getMRI()).getScalarType().isScalar() && "invalid offset type");
234
235 return buildInstr(Opc: TargetOpcode::G_PTR_ADD, DstOps: {Res}, SrcOps: {Op0, Op1}, Flags);
236}
237
238MachineInstrBuilder MachineIRBuilder::buildObjectPtrOffset(const DstOp &Res,
239 const SrcOp &Op0,
240 const SrcOp &Op1) {
241 return buildPtrAdd(Res, Op0, Op1,
242 Flags: MachineInstr::MIFlag::NoUWrap |
243 MachineInstr::MIFlag::InBounds);
244}
245
246std::optional<MachineInstrBuilder>
247MachineIRBuilder::materializePtrAdd(Register &Res, Register Op0,
248 const LLT ValueTy, uint64_t Value,
249 std::optional<unsigned> Flags) {
250 assert(Res == 0 && "Res is a result argument");
251 assert(ValueTy.isScalar() && "invalid offset type");
252
253 if (Value == 0) {
254 Res = Op0;
255 return std::nullopt;
256 }
257
258 Res = getMRI()->createGenericVirtualRegister(Ty: getMRI()->getType(Reg: Op0));
259 auto Cst = buildConstant(Res: ValueTy, Val: Value);
260 return buildPtrAdd(Res, Op0, Op1: Cst.getReg(Idx: 0), Flags);
261}
262
263std::optional<MachineInstrBuilder> MachineIRBuilder::materializeObjectPtrOffset(
264 Register &Res, Register Op0, const LLT ValueTy, uint64_t Value) {
265 return materializePtrAdd(Res, Op0, ValueTy, Value,
266 Flags: MachineInstr::MIFlag::NoUWrap |
267 MachineInstr::MIFlag::InBounds);
268}
269
270MachineInstrBuilder MachineIRBuilder::buildMaskLowPtrBits(const DstOp &Res,
271 const SrcOp &Op0,
272 uint32_t NumBits) {
273 LLT PtrTy = Res.getLLTTy(MRI: *getMRI());
274 LLT MaskTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
275 Register MaskReg = getMRI()->createGenericVirtualRegister(Ty: MaskTy);
276 buildConstant(Res: MaskReg, Val: maskTrailingZeros<uint64_t>(N: NumBits));
277 return buildPtrMask(Res, Op0, Op1: MaskReg);
278}
279
280MachineInstrBuilder
281MachineIRBuilder::buildPadVectorWithUndefElements(const DstOp &Res,
282 const SrcOp &Op0) {
283 LLT ResTy = Res.getLLTTy(MRI: *getMRI());
284 LLT Op0Ty = Op0.getLLTTy(MRI: *getMRI());
285
286 assert(ResTy.isVector() && "Res non vector type");
287
288 SmallVector<Register, 8> Regs;
289 if (Op0Ty.isVector()) {
290 assert((ResTy.getElementType() == Op0Ty.getElementType()) &&
291 "Different vector element types");
292 assert((ResTy.getNumElements() > Op0Ty.getNumElements()) &&
293 "Op0 has more elements");
294 auto Unmerge = buildUnmerge(Res: Op0Ty.getElementType(), Op: Op0);
295
296 for (auto Op : Unmerge.getInstr()->defs())
297 Regs.push_back(Elt: Op.getReg());
298 } else {
299 assert((ResTy.getSizeInBits() > Op0Ty.getSizeInBits()) &&
300 "Op0 has more size");
301 Regs.push_back(Elt: Op0.getReg());
302 }
303 Register Undef =
304 buildUndef(Res: Op0Ty.isVector() ? Op0Ty.getElementType() : Op0Ty).getReg(Idx: 0);
305 unsigned NumberOfPadElts = ResTy.getNumElements() - Regs.size();
306 for (unsigned i = 0; i < NumberOfPadElts; ++i)
307 Regs.push_back(Elt: Undef);
308 return buildMergeLikeInstr(Res, Ops: Regs);
309}
310
311MachineInstrBuilder
312MachineIRBuilder::buildDeleteTrailingVectorElements(const DstOp &Res,
313 const SrcOp &Op0) {
314 LLT ResTy = Res.getLLTTy(MRI: *getMRI());
315 LLT Op0Ty = Op0.getLLTTy(MRI: *getMRI());
316
317 assert(Op0Ty.isVector() && "Non vector type");
318 assert(((ResTy.isScalar() && (ResTy == Op0Ty.getElementType())) ||
319 (ResTy.isVector() &&
320 (ResTy.getElementType() == Op0Ty.getElementType()))) &&
321 "Different vector element types");
322 assert(
323 (ResTy.isScalar() || (ResTy.getNumElements() < Op0Ty.getNumElements())) &&
324 "Op0 has fewer elements");
325
326 auto Unmerge = buildUnmerge(Res: Op0Ty.getElementType(), Op: Op0);
327 if (ResTy.isScalar())
328 return buildCopy(Res, Op: Unmerge.getReg(Idx: 0));
329 SmallVector<Register, 8> Regs;
330 for (unsigned i = 0; i < ResTy.getNumElements(); ++i)
331 Regs.push_back(Elt: Unmerge.getReg(Idx: i));
332 return buildMergeLikeInstr(Res, Ops: Regs);
333}
334
335MachineInstrBuilder MachineIRBuilder::buildBr(MachineBasicBlock &Dest) {
336 return buildInstr(Opcode: TargetOpcode::G_BR).addMBB(MBB: &Dest);
337}
338
339MachineInstrBuilder MachineIRBuilder::buildBrIndirect(Register Tgt) {
340 assert(getMRI()->getType(Tgt).isPointer() && "invalid branch destination");
341 return buildInstr(Opcode: TargetOpcode::G_BRINDIRECT).addUse(RegNo: Tgt);
342}
343
344MachineInstrBuilder MachineIRBuilder::buildBrJT(Register TablePtr,
345 unsigned JTI,
346 Register IndexReg) {
347 assert(getMRI()->getType(TablePtr).isPointer() &&
348 "Table reg must be a pointer");
349 return buildInstr(Opcode: TargetOpcode::G_BRJT)
350 .addUse(RegNo: TablePtr)
351 .addJumpTableIndex(Idx: JTI)
352 .addUse(RegNo: IndexReg);
353}
354
355MachineInstrBuilder MachineIRBuilder::buildCopy(const DstOp &Res,
356 const SrcOp &Op) {
357 return buildInstr(Opc: TargetOpcode::COPY, DstOps: Res, SrcOps: Op);
358}
359
360MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
361 const ConstantInt &Val) {
362 assert(!isa<VectorType>(Val.getType()) && "Unexpected vector constant!");
363 LLT Ty = Res.getLLTTy(MRI: *getMRI());
364 LLT EltTy = Ty.getScalarType();
365 assert(EltTy.getScalarSizeInBits() == Val.getBitWidth() &&
366 "creating constant with the wrong size");
367
368 assert(!Ty.isScalableVector() &&
369 "unexpected scalable vector in buildConstant");
370
371 if (Ty.isFixedVector()) {
372 auto Const = buildInstr(Opcode: TargetOpcode::G_CONSTANT)
373 .addDef(RegNo: getMRI()->createGenericVirtualRegister(Ty: EltTy))
374 .addCImm(Val: &Val);
375 return buildSplatBuildVector(Res, Src: Const);
376 }
377
378 auto Const = buildInstr(Opcode: TargetOpcode::G_CONSTANT);
379 Const->setDebugLoc(DebugLoc());
380 Res.addDefToMIB(MRI&: *getMRI(), MIB&: Const);
381 Const.addCImm(Val: &Val);
382 return Const;
383}
384
385MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
386 int64_t Val) {
387 auto IntN = IntegerType::get(C&: getMF().getFunction().getContext(),
388 NumBits: Res.getLLTTy(MRI: *getMRI()).getScalarSizeInBits());
389 // TODO: Avoid implicit trunc?
390 // See https://github.com/llvm/llvm-project/issues/112510.
391 ConstantInt *CI = ConstantInt::getSigned(Ty: IntN, V: Val, /*implicitTrunc=*/ImplicitTrunc: true);
392 return buildConstant(Res, Val: *CI);
393}
394
395MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
396 const ConstantFP &Val) {
397 assert(!isa<VectorType>(Val.getType()) && "Unexpected vector constant!");
398 LLT Ty = Res.getLLTTy(MRI: *getMRI());
399 LLT EltTy = Ty.getScalarType();
400
401 assert(APFloat::getSizeInBits(Val.getValueAPF().getSemantics())
402 == EltTy.getSizeInBits() &&
403 "creating fconstant with the wrong size");
404
405 assert(!Ty.isPointer() && "invalid operand type");
406
407 assert(!Ty.isScalableVector() &&
408 "unexpected scalable vector in buildFConstant");
409
410 if (Ty.isFixedVector()) {
411 auto Const = buildInstr(Opcode: TargetOpcode::G_FCONSTANT)
412 .addDef(RegNo: getMRI()->createGenericVirtualRegister(Ty: EltTy))
413 .addFPImm(Val: &Val);
414
415 return buildSplatBuildVector(Res, Src: Const);
416 }
417
418 auto Const = buildInstr(Opcode: TargetOpcode::G_FCONSTANT);
419 Const->setDebugLoc(DebugLoc());
420 Res.addDefToMIB(MRI&: *getMRI(), MIB&: Const);
421 Const.addFPImm(Val: &Val);
422 return Const;
423}
424
425MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
426 const APInt &Val) {
427 ConstantInt *CI = ConstantInt::get(Context&: getMF().getFunction().getContext(), V: Val);
428 return buildConstant(Res, Val: *CI);
429}
430
431MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
432 double Val) {
433 LLT DstTy = Res.getLLTTy(MRI: *getMRI());
434 auto &Ctx = getMF().getFunction().getContext();
435 APFloat APF(Val);
436 bool Ignored;
437 APF.convert(ToSemantics: getFltSemanticForLLT(Ty: DstTy.getScalarType()),
438 RM: APFloat::rmNearestTiesToEven, losesInfo: &Ignored);
439 return buildFConstant(Res, Val: *ConstantFP::get(Context&: Ctx, V: APF));
440}
441
442MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
443 const APFloat &Val) {
444 auto &Ctx = getMF().getFunction().getContext();
445 auto *CFP = ConstantFP::get(Context&: Ctx, V: Val);
446 return buildFConstant(Res, Val: *CFP);
447}
448
449MachineInstrBuilder
450MachineIRBuilder::buildConstantPtrAuth(const DstOp &Res,
451 const ConstantPtrAuth *CPA,
452 Register Addr, Register AddrDisc) {
453 auto MIB = buildInstr(Opcode: TargetOpcode::G_PTRAUTH_GLOBAL_VALUE);
454 Res.addDefToMIB(MRI&: *getMRI(), MIB);
455 MIB.addUse(RegNo: Addr);
456 MIB.addImm(Val: CPA->getKey()->getZExtValue());
457 MIB.addUse(RegNo: AddrDisc);
458 MIB.addImm(Val: CPA->getDiscriminator()->getZExtValue());
459 return MIB;
460}
461
462MachineInstrBuilder MachineIRBuilder::buildBrCond(const SrcOp &Tst,
463 MachineBasicBlock &Dest) {
464 assert(Tst.getLLTTy(*getMRI()).isScalar() && "invalid operand type");
465
466 auto MIB = buildInstr(Opcode: TargetOpcode::G_BRCOND);
467 Tst.addSrcToMIB(MIB);
468 MIB.addMBB(MBB: &Dest);
469 return MIB;
470}
471
472MachineInstrBuilder
473MachineIRBuilder::buildLoad(const DstOp &Dst, const SrcOp &Addr,
474 MachinePointerInfo PtrInfo, Align Alignment,
475 MachineMemOperand::Flags MMOFlags,
476 const AAMDNodes &AAInfo) {
477 MMOFlags |= MachineMemOperand::MOLoad;
478 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
479
480 LLT Ty = Dst.getLLTTy(MRI: *getMRI());
481 MachineMemOperand *MMO =
482 getMF().getMachineMemOperand(PtrInfo, F: MMOFlags, MemTy: Ty, BaseAlignment: Alignment, Metadata: AAInfo);
483 return buildLoad(Res: Dst, Addr, MMO&: *MMO);
484}
485
486MachineInstrBuilder MachineIRBuilder::buildLoadInstr(unsigned Opcode,
487 const DstOp &Res,
488 const SrcOp &Addr,
489 MachineMemOperand &MMO) {
490 assert(Res.getLLTTy(*getMRI()).isValid() && "invalid operand type");
491 assert(Addr.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
492
493 auto MIB = buildInstr(Opcode);
494 Res.addDefToMIB(MRI&: *getMRI(), MIB);
495 Addr.addSrcToMIB(MIB);
496 MIB.addMemOperand(MMO: &MMO);
497 return MIB;
498}
499
500MachineInstrBuilder MachineIRBuilder::buildLoadFromOffset(
501 const DstOp &Dst, const SrcOp &BasePtr,
502 MachineMemOperand &BaseMMO, int64_t Offset) {
503 LLT LoadTy = Dst.getLLTTy(MRI: *getMRI());
504 MachineMemOperand *OffsetMMO =
505 getMF().getMachineMemOperand(MMO: &BaseMMO, Offset, Ty: LoadTy);
506
507 if (Offset == 0) // This may be a size or type changing load.
508 return buildLoad(Res: Dst, Addr: BasePtr, MMO&: *OffsetMMO);
509
510 LLT PtrTy = BasePtr.getLLTTy(MRI: *getMRI());
511 LLT OffsetTy = LLT::scalar(SizeInBits: PtrTy.getSizeInBits());
512 auto ConstOffset = buildConstant(Res: OffsetTy, Val: Offset);
513 auto Ptr = buildPtrAdd(Res: PtrTy, Op0: BasePtr, Op1: ConstOffset);
514 return buildLoad(Res: Dst, Addr: Ptr, MMO&: *OffsetMMO);
515}
516
517MachineInstrBuilder MachineIRBuilder::buildStore(const SrcOp &Val,
518 const SrcOp &Addr,
519 MachineMemOperand &MMO) {
520 assert(Val.getLLTTy(*getMRI()).isValid() && "invalid operand type");
521 assert(Addr.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
522
523 auto MIB = buildInstr(Opcode: TargetOpcode::G_STORE);
524 Val.addSrcToMIB(MIB);
525 Addr.addSrcToMIB(MIB);
526 MIB.addMemOperand(MMO: &MMO);
527 return MIB;
528}
529
530MachineInstrBuilder MachineIRBuilder::buildStoreInstr(unsigned Opcode,
531 const SrcOp &Val,
532 const SrcOp &Addr,
533 MachineMemOperand &MMO) {
534 assert(Val.getLLTTy(*getMRI()).isValid() && "invalid operand type");
535 assert(Addr.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
536
537 auto MIB = buildInstr(Opcode);
538 Val.addSrcToMIB(MIB);
539 Addr.addSrcToMIB(MIB);
540 MIB.addMemOperand(MMO: &MMO);
541 return MIB;
542}
543
544MachineInstrBuilder
545MachineIRBuilder::buildStore(const SrcOp &Val, const SrcOp &Addr,
546 MachinePointerInfo PtrInfo, Align Alignment,
547 MachineMemOperand::Flags MMOFlags,
548 const AAMDNodes &AAInfo) {
549 MMOFlags |= MachineMemOperand::MOStore;
550 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
551
552 LLT Ty = Val.getLLTTy(MRI: *getMRI());
553 MachineMemOperand *MMO =
554 getMF().getMachineMemOperand(PtrInfo, F: MMOFlags, MemTy: Ty, BaseAlignment: Alignment, Metadata: AAInfo);
555 return buildStore(Val, Addr, MMO&: *MMO);
556}
557
558MachineInstrBuilder MachineIRBuilder::buildAnyExt(const DstOp &Res,
559 const SrcOp &Op) {
560 return buildInstr(Opc: TargetOpcode::G_ANYEXT, DstOps: Res, SrcOps: Op);
561}
562
563MachineInstrBuilder MachineIRBuilder::buildSExt(const DstOp &Res,
564 const SrcOp &Op) {
565 return buildInstr(Opc: TargetOpcode::G_SEXT, DstOps: Res, SrcOps: Op);
566}
567
568MachineInstrBuilder MachineIRBuilder::buildZExt(const DstOp &Res,
569 const SrcOp &Op,
570 std::optional<unsigned> Flags) {
571 return buildInstr(Opc: TargetOpcode::G_ZEXT, DstOps: Res, SrcOps: Op, Flags);
572}
573
574unsigned MachineIRBuilder::getBoolExtOp(bool IsVec, bool IsFP) const {
575 const auto *TLI = getMF().getSubtarget().getTargetLowering();
576 switch (TLI->getBooleanContents(isVec: IsVec, isFloat: IsFP)) {
577 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
578 return TargetOpcode::G_SEXT;
579 case TargetLoweringBase::ZeroOrOneBooleanContent:
580 return TargetOpcode::G_ZEXT;
581 default:
582 return TargetOpcode::G_ANYEXT;
583 }
584}
585
586MachineInstrBuilder MachineIRBuilder::buildBoolExt(const DstOp &Res,
587 const SrcOp &Op,
588 bool IsFP) {
589 unsigned ExtOp = getBoolExtOp(IsVec: getMRI()->getType(Reg: Op.getReg()).isVector(), IsFP);
590 return buildInstr(Opc: ExtOp, DstOps: Res, SrcOps: Op);
591}
592
593MachineInstrBuilder MachineIRBuilder::buildBoolExtInReg(const DstOp &Res,
594 const SrcOp &Op,
595 bool IsVector,
596 bool IsFP) {
597 const auto *TLI = getMF().getSubtarget().getTargetLowering();
598 switch (TLI->getBooleanContents(isVec: IsVector, isFloat: IsFP)) {
599 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
600 return buildSExtInReg(Res, Op, ImmOp: 1);
601 case TargetLoweringBase::ZeroOrOneBooleanContent:
602 return buildZExtInReg(Res, Op, ImmOp: 1);
603 case TargetLoweringBase::UndefinedBooleanContent:
604 return buildCopy(Res, Op);
605 }
606
607 llvm_unreachable("unexpected BooleanContent");
608}
609
610MachineInstrBuilder MachineIRBuilder::buildExtOrTrunc(unsigned ExtOpc,
611 const DstOp &Res,
612 const SrcOp &Op) {
613 assert((TargetOpcode::G_ANYEXT == ExtOpc || TargetOpcode::G_ZEXT == ExtOpc ||
614 TargetOpcode::G_SEXT == ExtOpc) &&
615 "Expecting Extending Opc");
616 assert(Res.getLLTTy(*getMRI()).isScalar() ||
617 Res.getLLTTy(*getMRI()).isVector());
618 assert(Res.getLLTTy(*getMRI()).isScalar() ==
619 Op.getLLTTy(*getMRI()).isScalar());
620
621 unsigned Opcode = TargetOpcode::COPY;
622 if (Res.getLLTTy(MRI: *getMRI()).getSizeInBits() >
623 Op.getLLTTy(MRI: *getMRI()).getSizeInBits())
624 Opcode = ExtOpc;
625 else if (Res.getLLTTy(MRI: *getMRI()).getSizeInBits() <
626 Op.getLLTTy(MRI: *getMRI()).getSizeInBits())
627 Opcode = TargetOpcode::G_TRUNC;
628 else
629 assert(Res.getLLTTy(*getMRI()).getSizeInBits() ==
630 Op.getLLTTy(*getMRI()).getSizeInBits());
631
632 return buildInstr(Opc: Opcode, DstOps: Res, SrcOps: Op);
633}
634
635MachineInstrBuilder MachineIRBuilder::buildSExtOrTrunc(const DstOp &Res,
636 const SrcOp &Op) {
637 return buildExtOrTrunc(ExtOpc: TargetOpcode::G_SEXT, Res, Op);
638}
639
640MachineInstrBuilder MachineIRBuilder::buildZExtOrTrunc(const DstOp &Res,
641 const SrcOp &Op) {
642 return buildExtOrTrunc(ExtOpc: TargetOpcode::G_ZEXT, Res, Op);
643}
644
645MachineInstrBuilder MachineIRBuilder::buildAnyExtOrTrunc(const DstOp &Res,
646 const SrcOp &Op) {
647 return buildExtOrTrunc(ExtOpc: TargetOpcode::G_ANYEXT, Res, Op);
648}
649
650MachineInstrBuilder MachineIRBuilder::buildZExtInReg(const DstOp &Res,
651 const SrcOp &Op,
652 int64_t ImmOp) {
653 LLT ResTy = Res.getLLTTy(MRI: *getMRI());
654 auto Mask = buildConstant(
655 Res: ResTy, Val: APInt::getLowBitsSet(numBits: ResTy.getScalarSizeInBits(), loBitsSet: ImmOp));
656 return buildAnd(Dst: Res, Src0: Op, Src1: Mask);
657}
658
659MachineInstrBuilder MachineIRBuilder::buildCast(const DstOp &Dst,
660 const SrcOp &Src) {
661 LLT SrcTy = Src.getLLTTy(MRI: *getMRI());
662 LLT DstTy = Dst.getLLTTy(MRI: *getMRI());
663 if (SrcTy == DstTy)
664 return buildCopy(Res: Dst, Op: Src);
665
666 unsigned Opcode;
667 if (SrcTy.isPointerOrPointerVector())
668 Opcode = TargetOpcode::G_PTRTOINT;
669 else if (DstTy.isPointerOrPointerVector())
670 Opcode = TargetOpcode::G_INTTOPTR;
671 else {
672 assert(!SrcTy.isPointerOrPointerVector() &&
673 !DstTy.isPointerOrPointerVector() && "no G_ADDRCAST yet");
674 Opcode = TargetOpcode::G_BITCAST;
675 }
676
677 return buildInstr(Opc: Opcode, DstOps: Dst, SrcOps: Src);
678}
679
680MachineInstrBuilder MachineIRBuilder::buildExtract(const DstOp &Dst,
681 const SrcOp &Src,
682 uint64_t Index) {
683 LLT SrcTy = Src.getLLTTy(MRI: *getMRI());
684 LLT DstTy = Dst.getLLTTy(MRI: *getMRI());
685
686#ifndef NDEBUG
687 assert(SrcTy.isValid() && "invalid operand type");
688 assert(DstTy.isValid() && "invalid operand type");
689 assert(Index + DstTy.getSizeInBits() <= SrcTy.getSizeInBits() &&
690 "extracting off end of register");
691#endif
692
693 if (DstTy.getSizeInBits() == SrcTy.getSizeInBits()) {
694 assert(Index == 0 && "insertion past the end of a register");
695 return buildCast(Dst, Src);
696 }
697
698 auto Extract = buildInstr(Opcode: TargetOpcode::G_EXTRACT);
699 Dst.addDefToMIB(MRI&: *getMRI(), MIB&: Extract);
700 Src.addSrcToMIB(MIB&: Extract);
701 Extract.addImm(Val: Index);
702 return Extract;
703}
704
705MachineInstrBuilder MachineIRBuilder::buildUndef(const DstOp &Res) {
706 return buildInstr(Opc: TargetOpcode::G_IMPLICIT_DEF, DstOps: {Res}, SrcOps: {});
707}
708
709MachineInstrBuilder MachineIRBuilder::buildMergeValues(const DstOp &Res,
710 ArrayRef<Register> Ops) {
711 // Unfortunately to convert from ArrayRef<LLT> to ArrayRef<SrcOp>,
712 // we need some temporary storage for the DstOp objects. Here we use a
713 // sufficiently large SmallVector to not go through the heap.
714 SmallVector<SrcOp, 8> TmpVec(Ops);
715 assert(TmpVec.size() > 1);
716 return buildInstr(Opc: TargetOpcode::G_MERGE_VALUES, DstOps: Res, SrcOps: TmpVec);
717}
718
719MachineInstrBuilder
720MachineIRBuilder::buildMergeLikeInstr(const DstOp &Res,
721 ArrayRef<Register> Ops) {
722 // Unfortunately to convert from ArrayRef<LLT> to ArrayRef<SrcOp>,
723 // we need some temporary storage for the DstOp objects. Here we use a
724 // sufficiently large SmallVector to not go through the heap.
725 SmallVector<SrcOp, 8> TmpVec(Ops);
726 assert(TmpVec.size() > 1);
727 return buildInstr(Opc: getOpcodeForMerge(DstOp: Res, SrcOps: TmpVec), DstOps: Res, SrcOps: TmpVec);
728}
729
730MachineInstrBuilder
731MachineIRBuilder::buildMergeLikeInstr(const DstOp &Res,
732 std::initializer_list<SrcOp> Ops) {
733 assert(Ops.size() > 1);
734 return buildInstr(Opc: getOpcodeForMerge(DstOp: Res, SrcOps: Ops), DstOps: Res, SrcOps: Ops);
735}
736
737unsigned MachineIRBuilder::getOpcodeForMerge(const DstOp &DstOp,
738 ArrayRef<SrcOp> SrcOps) const {
739 if (DstOp.getLLTTy(MRI: *getMRI()).isVector()) {
740 if (SrcOps[0].getLLTTy(MRI: *getMRI()).isVector())
741 return TargetOpcode::G_CONCAT_VECTORS;
742 return TargetOpcode::G_BUILD_VECTOR;
743 }
744
745 return TargetOpcode::G_MERGE_VALUES;
746}
747
748MachineInstrBuilder MachineIRBuilder::buildUnmerge(ArrayRef<LLT> Res,
749 const SrcOp &Op) {
750 // Unfortunately to convert from ArrayRef<LLT> to ArrayRef<DstOp>,
751 // we need some temporary storage for the DstOp objects. Here we use a
752 // sufficiently large SmallVector to not go through the heap.
753 SmallVector<DstOp, 8> TmpVec(Res);
754 assert(TmpVec.size() > 1);
755 return buildInstr(Opc: TargetOpcode::G_UNMERGE_VALUES, DstOps: TmpVec, SrcOps: Op);
756}
757
758MachineInstrBuilder MachineIRBuilder::buildUnmerge(LLT Res,
759 const SrcOp &Op) {
760 unsigned NumReg = Op.getLLTTy(MRI: *getMRI()).getSizeInBits() / Res.getSizeInBits();
761 SmallVector<DstOp, 8> TmpVec(NumReg, Res);
762 return buildInstr(Opc: TargetOpcode::G_UNMERGE_VALUES, DstOps: TmpVec, SrcOps: Op);
763}
764
765MachineInstrBuilder
766MachineIRBuilder::buildUnmerge(MachineRegisterInfo::VRegAttrs Attrs,
767 const SrcOp &Op) {
768 LLT OpTy = Op.getLLTTy(MRI: *getMRI());
769 unsigned NumRegs = OpTy.getSizeInBits() / Attrs.Ty.getSizeInBits();
770 SmallVector<DstOp, 8> TmpVec(NumRegs, Attrs);
771 return buildInstr(Opc: TargetOpcode::G_UNMERGE_VALUES, DstOps: TmpVec, SrcOps: Op);
772}
773
774MachineInstrBuilder MachineIRBuilder::buildUnmerge(ArrayRef<Register> Res,
775 const SrcOp &Op) {
776 // Unfortunately to convert from ArrayRef<Register> to ArrayRef<DstOp>,
777 // we need some temporary storage for the DstOp objects. Here we use a
778 // sufficiently large SmallVector to not go through the heap.
779 SmallVector<DstOp, 8> TmpVec(Res);
780 assert(TmpVec.size() > 1);
781 return buildInstr(Opc: TargetOpcode::G_UNMERGE_VALUES, DstOps: TmpVec, SrcOps: Op);
782}
783
784MachineInstrBuilder MachineIRBuilder::buildBuildVector(const DstOp &Res,
785 ArrayRef<Register> Ops) {
786 // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
787 // we need some temporary storage for the DstOp objects. Here we use a
788 // sufficiently large SmallVector to not go through the heap.
789 SmallVector<SrcOp, 8> TmpVec(Ops);
790 return buildInstr(Opc: TargetOpcode::G_BUILD_VECTOR, DstOps: Res, SrcOps: TmpVec);
791}
792
793MachineInstrBuilder
794MachineIRBuilder::buildBuildVectorConstant(const DstOp &Res,
795 ArrayRef<APInt> Ops) {
796 SmallVector<SrcOp> TmpVec;
797 TmpVec.reserve(N: Ops.size());
798 LLT EltTy = Res.getLLTTy(MRI: *getMRI()).getElementType();
799 for (const auto &Op : Ops)
800 TmpVec.push_back(Elt: buildConstant(Res: EltTy, Val: Op));
801 return buildInstr(Opc: TargetOpcode::G_BUILD_VECTOR, DstOps: Res, SrcOps: TmpVec);
802}
803
804MachineInstrBuilder MachineIRBuilder::buildSplatBuildVector(const DstOp &Res,
805 const SrcOp &Src) {
806 SmallVector<SrcOp, 8> TmpVec(Res.getLLTTy(MRI: *getMRI()).getNumElements(), Src);
807 return buildInstr(Opc: TargetOpcode::G_BUILD_VECTOR, DstOps: Res, SrcOps: TmpVec);
808}
809
810MachineInstrBuilder
811MachineIRBuilder::buildBuildVectorTrunc(const DstOp &Res,
812 ArrayRef<Register> Ops) {
813 // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
814 // we need some temporary storage for the DstOp objects. Here we use a
815 // sufficiently large SmallVector to not go through the heap.
816 SmallVector<SrcOp, 8> TmpVec(Ops);
817 if (TmpVec[0].getLLTTy(MRI: *getMRI()).getSizeInBits() ==
818 Res.getLLTTy(MRI: *getMRI()).getElementType().getSizeInBits())
819 return buildInstr(Opc: TargetOpcode::G_BUILD_VECTOR, DstOps: Res, SrcOps: TmpVec);
820 return buildInstr(Opc: TargetOpcode::G_BUILD_VECTOR_TRUNC, DstOps: Res, SrcOps: TmpVec);
821}
822
823MachineInstrBuilder MachineIRBuilder::buildShuffleSplat(const DstOp &Res,
824 const SrcOp &Src) {
825 LLT DstTy = Res.getLLTTy(MRI: *getMRI());
826 assert(Src.getLLTTy(*getMRI()) == DstTy.getElementType() &&
827 "Expected Src to match Dst elt ty");
828 auto UndefVec = buildUndef(Res: DstTy);
829 auto Zero = buildConstant(Res: LLT::integer(SizeInBits: 64), Val: 0);
830 auto InsElt = buildInsertVectorElement(Res: DstTy, Val: UndefVec, Elt: Src, Idx: Zero);
831 SmallVector<int, 16> ZeroMask(DstTy.getNumElements());
832 return buildShuffleVector(Res: DstTy, Src1: InsElt, Src2: UndefVec, Mask: ZeroMask);
833}
834
835MachineInstrBuilder MachineIRBuilder::buildSplatVector(const DstOp &Res,
836 const SrcOp &Src) {
837 assert(Src.getLLTTy(*getMRI()) == Res.getLLTTy(*getMRI()).getElementType() &&
838 "Expected Src to match Dst elt ty");
839 return buildInstr(Opc: TargetOpcode::G_SPLAT_VECTOR, DstOps: Res, SrcOps: Src);
840}
841
842MachineInstrBuilder MachineIRBuilder::buildShuffleVector(const DstOp &Res,
843 const SrcOp &Src1,
844 const SrcOp &Src2,
845 ArrayRef<int> Mask) {
846 LLT DstTy = Res.getLLTTy(MRI: *getMRI());
847 LLT Src1Ty = Src1.getLLTTy(MRI: *getMRI());
848 LLT Src2Ty = Src2.getLLTTy(MRI: *getMRI());
849 const LLT DstElemTy = DstTy.getScalarType();
850 const LLT ElemTy1 = Src1Ty.getScalarType();
851 const LLT ElemTy2 = Src2Ty.getScalarType();
852 assert(DstElemTy == ElemTy1 && DstElemTy == ElemTy2);
853 assert(Mask.size() > 1 && "Scalar G_SHUFFLE_VECTOR are not supported");
854 (void)DstElemTy;
855 (void)ElemTy1;
856 (void)ElemTy2;
857 ArrayRef<int> MaskAlloc = getMF().allocateShuffleMask(Mask);
858 return buildInstr(Opc: TargetOpcode::G_SHUFFLE_VECTOR, DstOps: {Res}, SrcOps: {Src1, Src2})
859 .addShuffleMask(Val: MaskAlloc);
860}
861
862MachineInstrBuilder
863MachineIRBuilder::buildConcatVectors(const DstOp &Res, ArrayRef<Register> Ops) {
864 // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
865 // we need some temporary storage for the DstOp objects. Here we use a
866 // sufficiently large SmallVector to not go through the heap.
867 SmallVector<SrcOp, 8> TmpVec(Ops);
868 return buildInstr(Opc: TargetOpcode::G_CONCAT_VECTORS, DstOps: Res, SrcOps: TmpVec);
869}
870
871MachineInstrBuilder MachineIRBuilder::buildInsert(const DstOp &Res,
872 const SrcOp &Src,
873 const SrcOp &Op,
874 unsigned Index) {
875 assert(Index + Op.getLLTTy(*getMRI()).getSizeInBits() <=
876 Res.getLLTTy(*getMRI()).getSizeInBits() &&
877 "insertion past the end of a register");
878
879 if (Res.getLLTTy(MRI: *getMRI()).getSizeInBits() ==
880 Op.getLLTTy(MRI: *getMRI()).getSizeInBits()) {
881 return buildCast(Dst: Res, Src: Op);
882 }
883
884 return buildInstr(Opc: TargetOpcode::G_INSERT, DstOps: Res, SrcOps: {Src, Op, uint64_t(Index)});
885}
886
887MachineInstrBuilder MachineIRBuilder::buildStepVector(const DstOp &Res,
888 unsigned Step) {
889 unsigned Bitwidth = Res.getLLTTy(MRI: *getMRI()).getElementType().getSizeInBits();
890 ConstantInt *CI = ConstantInt::get(Context&: getMF().getFunction().getContext(),
891 V: APInt(Bitwidth, Step));
892 auto StepVector = buildInstr(Opcode: TargetOpcode::G_STEP_VECTOR);
893 StepVector->setDebugLoc(DebugLoc());
894 Res.addDefToMIB(MRI&: *getMRI(), MIB&: StepVector);
895 StepVector.addCImm(Val: CI);
896 return StepVector;
897}
898
899MachineInstrBuilder MachineIRBuilder::buildVScale(const DstOp &Res,
900 unsigned MinElts) {
901
902 auto IntN = IntegerType::get(C&: getMF().getFunction().getContext(),
903 NumBits: Res.getLLTTy(MRI: *getMRI()).getScalarSizeInBits());
904 ConstantInt *CI = ConstantInt::get(Ty: IntN, V: MinElts);
905 return buildVScale(Res, MinElts: *CI);
906}
907
908MachineInstrBuilder MachineIRBuilder::buildVScale(const DstOp &Res,
909 const ConstantInt &MinElts) {
910 auto VScale = buildInstr(Opcode: TargetOpcode::G_VSCALE);
911 VScale->setDebugLoc(DebugLoc());
912 Res.addDefToMIB(MRI&: *getMRI(), MIB&: VScale);
913 VScale.addCImm(Val: &MinElts);
914 return VScale;
915}
916
917MachineInstrBuilder MachineIRBuilder::buildVScale(const DstOp &Res,
918 const APInt &MinElts) {
919 ConstantInt *CI =
920 ConstantInt::get(Context&: getMF().getFunction().getContext(), V: MinElts);
921 return buildVScale(Res, MinElts: *CI);
922}
923
924static unsigned getIntrinsicOpcode(bool HasSideEffects, bool IsConvergent) {
925 if (HasSideEffects && IsConvergent)
926 return TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS;
927 if (HasSideEffects)
928 return TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS;
929 if (IsConvergent)
930 return TargetOpcode::G_INTRINSIC_CONVERGENT;
931 return TargetOpcode::G_INTRINSIC;
932}
933
934MachineInstrBuilder
935MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
936 ArrayRef<Register> ResultRegs,
937 bool HasSideEffects, bool isConvergent) {
938 auto MIB = buildInstr(Opcode: getIntrinsicOpcode(HasSideEffects, IsConvergent: isConvergent));
939 for (Register ResultReg : ResultRegs)
940 MIB.addDef(RegNo: ResultReg);
941 MIB.addIntrinsicID(ID);
942 return MIB;
943}
944
945MachineInstrBuilder
946MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
947 ArrayRef<Register> ResultRegs) {
948 AttributeSet Attrs = Intrinsic::getFnAttributes(C&: getContext(), id: ID);
949 bool HasSideEffects = !Attrs.getMemoryEffects().doesNotAccessMemory();
950 bool isConvergent = Attrs.hasAttribute(Kind: Attribute::Convergent);
951 return buildIntrinsic(ID, ResultRegs, HasSideEffects, isConvergent);
952}
953
954MachineInstrBuilder MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
955 ArrayRef<DstOp> Results,
956 bool HasSideEffects,
957 bool isConvergent) {
958 auto MIB = buildInstr(Opcode: getIntrinsicOpcode(HasSideEffects, IsConvergent: isConvergent));
959 for (DstOp Result : Results)
960 Result.addDefToMIB(MRI&: *getMRI(), MIB);
961 MIB.addIntrinsicID(ID);
962 return MIB;
963}
964
965MachineInstrBuilder MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
966 ArrayRef<DstOp> Results) {
967 AttributeSet Attrs = Intrinsic::getFnAttributes(C&: getContext(), id: ID);
968 bool HasSideEffects = !Attrs.getMemoryEffects().doesNotAccessMemory();
969 bool isConvergent = Attrs.hasAttribute(Kind: Attribute::Convergent);
970 return buildIntrinsic(ID, Results, HasSideEffects, isConvergent);
971}
972
973MachineInstrBuilder
974MachineIRBuilder::buildTrunc(const DstOp &Res, const SrcOp &Op,
975 std::optional<unsigned> Flags) {
976 return buildInstr(Opc: TargetOpcode::G_TRUNC, DstOps: Res, SrcOps: Op, Flags);
977}
978
979MachineInstrBuilder
980MachineIRBuilder::buildFPTrunc(const DstOp &Res, const SrcOp &Op,
981 std::optional<unsigned> Flags) {
982 return buildInstr(Opc: TargetOpcode::G_FPTRUNC, DstOps: Res, SrcOps: Op, Flags);
983}
984
985MachineInstrBuilder MachineIRBuilder::buildICmp(CmpInst::Predicate Pred,
986 const DstOp &Res,
987 const SrcOp &Op0,
988 const SrcOp &Op1,
989 std::optional<unsigned> Flags) {
990 return buildInstr(Opc: TargetOpcode::G_ICMP, DstOps: Res, SrcOps: {Pred, Op0, Op1}, Flags);
991}
992
993MachineInstrBuilder MachineIRBuilder::buildFCmp(CmpInst::Predicate Pred,
994 const DstOp &Res,
995 const SrcOp &Op0,
996 const SrcOp &Op1,
997 std::optional<unsigned> Flags) {
998
999 return buildInstr(Opc: TargetOpcode::G_FCMP, DstOps: Res, SrcOps: {Pred, Op0, Op1}, Flags);
1000}
1001
1002MachineInstrBuilder MachineIRBuilder::buildSCmp(const DstOp &Res,
1003 const SrcOp &Op0,
1004 const SrcOp &Op1) {
1005 return buildInstr(Opc: TargetOpcode::G_SCMP, DstOps: Res, SrcOps: {Op0, Op1});
1006}
1007
1008MachineInstrBuilder MachineIRBuilder::buildUCmp(const DstOp &Res,
1009 const SrcOp &Op0,
1010 const SrcOp &Op1) {
1011 return buildInstr(Opc: TargetOpcode::G_UCMP, DstOps: Res, SrcOps: {Op0, Op1});
1012}
1013
1014MachineInstrBuilder
1015MachineIRBuilder::buildSelect(const DstOp &Res, const SrcOp &Tst,
1016 const SrcOp &Op0, const SrcOp &Op1,
1017 std::optional<unsigned> Flags) {
1018
1019 return buildInstr(Opc: TargetOpcode::G_SELECT, DstOps: {Res}, SrcOps: {Tst, Op0, Op1}, Flags);
1020}
1021
1022MachineInstrBuilder MachineIRBuilder::buildInsertSubvector(const DstOp &Res,
1023 const SrcOp &Src0,
1024 const SrcOp &Src1,
1025 unsigned Idx) {
1026 return buildInstr(Opc: TargetOpcode::G_INSERT_SUBVECTOR, DstOps: Res,
1027 SrcOps: {Src0, Src1, uint64_t(Idx)});
1028}
1029
1030MachineInstrBuilder MachineIRBuilder::buildExtractSubvector(const DstOp &Res,
1031 const SrcOp &Src,
1032 unsigned Idx) {
1033 return buildInstr(Opc: TargetOpcode::G_EXTRACT_SUBVECTOR, DstOps: Res,
1034 SrcOps: {Src, uint64_t(Idx)});
1035}
1036
1037MachineInstrBuilder
1038MachineIRBuilder::buildInsertVectorElement(const DstOp &Res, const SrcOp &Val,
1039 const SrcOp &Elt, const SrcOp &Idx) {
1040 return buildInstr(Opc: TargetOpcode::G_INSERT_VECTOR_ELT, DstOps: Res, SrcOps: {Val, Elt, Idx});
1041}
1042
1043MachineInstrBuilder
1044MachineIRBuilder::buildExtractVectorElement(const DstOp &Res, const SrcOp &Val,
1045 const SrcOp &Idx) {
1046 return buildInstr(Opc: TargetOpcode::G_EXTRACT_VECTOR_ELT, DstOps: Res, SrcOps: {Val, Idx});
1047}
1048
1049MachineInstrBuilder MachineIRBuilder::buildAtomicCmpXchgWithSuccess(
1050 const DstOp &OldValRes, const DstOp &SuccessRes, const SrcOp &Addr,
1051 const SrcOp &CmpVal, const SrcOp &NewVal, MachineMemOperand &MMO) {
1052#ifndef NDEBUG
1053 LLT OldValResTy = OldValRes.getLLTTy(*getMRI());
1054 LLT SuccessResTy = SuccessRes.getLLTTy(*getMRI());
1055 LLT AddrTy = Addr.getLLTTy(*getMRI());
1056 LLT CmpValTy = CmpVal.getLLTTy(*getMRI());
1057 LLT NewValTy = NewVal.getLLTTy(*getMRI());
1058 assert(OldValResTy.isScalar() && "invalid operand type");
1059 assert(SuccessResTy.isScalar() && "invalid operand type");
1060 assert(AddrTy.isPointer() && "invalid operand type");
1061 assert(CmpValTy.isValid() && "invalid operand type");
1062 assert(NewValTy.isValid() && "invalid operand type");
1063 assert(OldValResTy == CmpValTy && "type mismatch");
1064 assert(OldValResTy == NewValTy && "type mismatch");
1065#endif
1066
1067 auto MIB = buildInstr(Opcode: TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS);
1068 OldValRes.addDefToMIB(MRI&: *getMRI(), MIB);
1069 SuccessRes.addDefToMIB(MRI&: *getMRI(), MIB);
1070 Addr.addSrcToMIB(MIB);
1071 CmpVal.addSrcToMIB(MIB);
1072 NewVal.addSrcToMIB(MIB);
1073 MIB.addMemOperand(MMO: &MMO);
1074 return MIB;
1075}
1076
1077MachineInstrBuilder
1078MachineIRBuilder::buildAtomicCmpXchg(const DstOp &OldValRes, const SrcOp &Addr,
1079 const SrcOp &CmpVal, const SrcOp &NewVal,
1080 MachineMemOperand &MMO) {
1081#ifndef NDEBUG
1082 LLT OldValResTy = OldValRes.getLLTTy(*getMRI());
1083 LLT AddrTy = Addr.getLLTTy(*getMRI());
1084 LLT CmpValTy = CmpVal.getLLTTy(*getMRI());
1085 LLT NewValTy = NewVal.getLLTTy(*getMRI());
1086 assert(OldValResTy.isScalar() && "invalid operand type");
1087 assert(AddrTy.isPointer() && "invalid operand type");
1088 assert(CmpValTy.isValid() && "invalid operand type");
1089 assert(NewValTy.isValid() && "invalid operand type");
1090 assert(OldValResTy == CmpValTy && "type mismatch");
1091 assert(OldValResTy == NewValTy && "type mismatch");
1092#endif
1093
1094 auto MIB = buildInstr(Opcode: TargetOpcode::G_ATOMIC_CMPXCHG);
1095 OldValRes.addDefToMIB(MRI&: *getMRI(), MIB);
1096 Addr.addSrcToMIB(MIB);
1097 CmpVal.addSrcToMIB(MIB);
1098 NewVal.addSrcToMIB(MIB);
1099 MIB.addMemOperand(MMO: &MMO);
1100 return MIB;
1101}
1102
1103MachineInstrBuilder MachineIRBuilder::buildAtomicRMW(
1104 unsigned Opcode, const DstOp &OldValRes,
1105 const SrcOp &Addr, const SrcOp &Val,
1106 MachineMemOperand &MMO) {
1107
1108#ifndef NDEBUG
1109 LLT OldValResTy = OldValRes.getLLTTy(*getMRI());
1110 LLT AddrTy = Addr.getLLTTy(*getMRI());
1111 LLT ValTy = Val.getLLTTy(*getMRI());
1112 assert(AddrTy.isPointer() && "invalid operand type");
1113 assert(ValTy.isValid() && "invalid operand type");
1114 assert(OldValResTy == ValTy && "type mismatch");
1115 assert(MMO.isAtomic() && "not atomic mem operand");
1116#endif
1117
1118 auto MIB = buildInstr(Opcode);
1119 OldValRes.addDefToMIB(MRI&: *getMRI(), MIB);
1120 Addr.addSrcToMIB(MIB);
1121 Val.addSrcToMIB(MIB);
1122 MIB.addMemOperand(MMO: &MMO);
1123 return MIB;
1124}
1125
1126MachineInstrBuilder
1127MachineIRBuilder::buildAtomicRMWXchg(Register OldValRes, Register Addr,
1128 Register Val, MachineMemOperand &MMO) {
1129 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_XCHG, OldValRes, Addr, Val,
1130 MMO);
1131}
1132MachineInstrBuilder
1133MachineIRBuilder::buildAtomicRMWAdd(Register OldValRes, Register Addr,
1134 Register Val, MachineMemOperand &MMO) {
1135 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_ADD, OldValRes, Addr, Val,
1136 MMO);
1137}
1138MachineInstrBuilder
1139MachineIRBuilder::buildAtomicRMWSub(Register OldValRes, Register Addr,
1140 Register Val, MachineMemOperand &MMO) {
1141 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_SUB, OldValRes, Addr, Val,
1142 MMO);
1143}
1144MachineInstrBuilder
1145MachineIRBuilder::buildAtomicRMWAnd(Register OldValRes, Register Addr,
1146 Register Val, MachineMemOperand &MMO) {
1147 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_AND, OldValRes, Addr, Val,
1148 MMO);
1149}
1150MachineInstrBuilder
1151MachineIRBuilder::buildAtomicRMWNand(Register OldValRes, Register Addr,
1152 Register Val, MachineMemOperand &MMO) {
1153 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_NAND, OldValRes, Addr, Val,
1154 MMO);
1155}
1156MachineInstrBuilder MachineIRBuilder::buildAtomicRMWOr(Register OldValRes,
1157 Register Addr,
1158 Register Val,
1159 MachineMemOperand &MMO) {
1160 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_OR, OldValRes, Addr, Val,
1161 MMO);
1162}
1163MachineInstrBuilder
1164MachineIRBuilder::buildAtomicRMWXor(Register OldValRes, Register Addr,
1165 Register Val, MachineMemOperand &MMO) {
1166 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_XOR, OldValRes, Addr, Val,
1167 MMO);
1168}
1169MachineInstrBuilder
1170MachineIRBuilder::buildAtomicRMWMax(Register OldValRes, Register Addr,
1171 Register Val, MachineMemOperand &MMO) {
1172 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_MAX, OldValRes, Addr, Val,
1173 MMO);
1174}
1175MachineInstrBuilder
1176MachineIRBuilder::buildAtomicRMWMin(Register OldValRes, Register Addr,
1177 Register Val, MachineMemOperand &MMO) {
1178 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_MIN, OldValRes, Addr, Val,
1179 MMO);
1180}
1181MachineInstrBuilder
1182MachineIRBuilder::buildAtomicRMWUmax(Register OldValRes, Register Addr,
1183 Register Val, MachineMemOperand &MMO) {
1184 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_UMAX, OldValRes, Addr, Val,
1185 MMO);
1186}
1187MachineInstrBuilder
1188MachineIRBuilder::buildAtomicRMWUmin(Register OldValRes, Register Addr,
1189 Register Val, MachineMemOperand &MMO) {
1190 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_UMIN, OldValRes, Addr, Val,
1191 MMO);
1192}
1193
1194MachineInstrBuilder
1195MachineIRBuilder::buildAtomicRMWFAdd(
1196 const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val,
1197 MachineMemOperand &MMO) {
1198 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_FADD, OldValRes, Addr, Val,
1199 MMO);
1200}
1201
1202MachineInstrBuilder
1203MachineIRBuilder::buildAtomicRMWFSub(const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val,
1204 MachineMemOperand &MMO) {
1205 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_FSUB, OldValRes, Addr, Val,
1206 MMO);
1207}
1208
1209MachineInstrBuilder
1210MachineIRBuilder::buildAtomicRMWFMax(const DstOp &OldValRes, const SrcOp &Addr,
1211 const SrcOp &Val, MachineMemOperand &MMO) {
1212 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_FMAX, OldValRes, Addr, Val,
1213 MMO);
1214}
1215
1216MachineInstrBuilder
1217MachineIRBuilder::buildAtomicRMWFMin(const DstOp &OldValRes, const SrcOp &Addr,
1218 const SrcOp &Val, MachineMemOperand &MMO) {
1219 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_FMIN, OldValRes, Addr, Val,
1220 MMO);
1221}
1222
1223MachineInstrBuilder
1224MachineIRBuilder::buildAtomicRMWFMaximum(const DstOp &OldValRes,
1225 const SrcOp &Addr, const SrcOp &Val,
1226 MachineMemOperand &MMO) {
1227 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_FMAXIMUM, OldValRes, Addr,
1228 Val, MMO);
1229}
1230
1231MachineInstrBuilder
1232MachineIRBuilder::buildAtomicRMWFMinimum(const DstOp &OldValRes,
1233 const SrcOp &Addr, const SrcOp &Val,
1234 MachineMemOperand &MMO) {
1235 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_FMINIMUM, OldValRes, Addr,
1236 Val, MMO);
1237}
1238
1239MachineInstrBuilder
1240MachineIRBuilder::buildFence(unsigned Ordering, unsigned Scope) {
1241 return buildInstr(Opcode: TargetOpcode::G_FENCE)
1242 .addImm(Val: Ordering)
1243 .addImm(Val: Scope);
1244}
1245
1246MachineInstrBuilder MachineIRBuilder::buildPrefetch(const SrcOp &Addr,
1247 unsigned RW,
1248 unsigned Locality,
1249 unsigned CacheType,
1250 MachineMemOperand &MMO) {
1251 auto MIB = buildInstr(Opcode: TargetOpcode::G_PREFETCH);
1252 Addr.addSrcToMIB(MIB);
1253 MIB.addImm(Val: RW).addImm(Val: Locality).addImm(Val: CacheType);
1254 MIB.addMemOperand(MMO: &MMO);
1255 return MIB;
1256}
1257
1258MachineInstrBuilder
1259MachineIRBuilder::buildBlockAddress(Register Res, const BlockAddress *BA) {
1260#ifndef NDEBUG
1261 assert(getMRI()->getType(Res).isPointer() && "invalid res type");
1262#endif
1263
1264 return buildInstr(Opcode: TargetOpcode::G_BLOCK_ADDR).addDef(RegNo: Res).addBlockAddress(BA);
1265}
1266
1267void MachineIRBuilder::validateTruncExt(const LLT DstTy, const LLT SrcTy,
1268 bool IsExtend) {
1269#ifndef NDEBUG
1270 if (DstTy.isVector()) {
1271 assert(SrcTy.isVector() && "mismatched cast between vector and non-vector");
1272 assert(SrcTy.getElementCount() == DstTy.getElementCount() &&
1273 "different number of elements in a trunc/ext");
1274 } else
1275 assert(DstTy.isScalar() && SrcTy.isScalar() && "invalid extend/trunc");
1276
1277 if (IsExtend)
1278 assert(TypeSize::isKnownGT(DstTy.getSizeInBits(), SrcTy.getSizeInBits()) &&
1279 "invalid narrowing extend");
1280 else
1281 assert(TypeSize::isKnownLT(DstTy.getSizeInBits(), SrcTy.getSizeInBits()) &&
1282 "invalid widening trunc");
1283#endif
1284}
1285
1286void MachineIRBuilder::validateSelectOp(const LLT ResTy, const LLT TstTy,
1287 const LLT Op0Ty, const LLT Op1Ty) {
1288#ifndef NDEBUG
1289 assert((ResTy.isScalar() || ResTy.isVector() || ResTy.isPointer()) &&
1290 "invalid operand type");
1291 assert((ResTy == Op0Ty && ResTy == Op1Ty) && "type mismatch");
1292 if (ResTy.isScalar() || ResTy.isPointer())
1293 assert(TstTy.isScalar() && "type mismatch");
1294 else
1295 assert((TstTy.isScalar() ||
1296 (TstTy.isVector() &&
1297 TstTy.getElementCount() == Op0Ty.getElementCount())) &&
1298 "type mismatch");
1299#endif
1300}
1301
1302MachineInstrBuilder
1303MachineIRBuilder::buildInstr(unsigned Opc, ArrayRef<DstOp> DstOps,
1304 ArrayRef<SrcOp> SrcOps,
1305 std::optional<unsigned> Flags) {
1306 switch (Opc) {
1307 default:
1308 break;
1309 case TargetOpcode::G_SELECT: {
1310 assert(DstOps.size() == 1 && "Invalid select");
1311 assert(SrcOps.size() == 3 && "Invalid select");
1312 validateSelectOp(
1313 ResTy: DstOps[0].getLLTTy(MRI: *getMRI()), TstTy: SrcOps[0].getLLTTy(MRI: *getMRI()),
1314 Op0Ty: SrcOps[1].getLLTTy(MRI: *getMRI()), Op1Ty: SrcOps[2].getLLTTy(MRI: *getMRI()));
1315 break;
1316 }
1317 case TargetOpcode::G_FNEG:
1318 case TargetOpcode::G_ABS:
1319 // All these are unary ops.
1320 assert(DstOps.size() == 1 && "Invalid Dst");
1321 assert(SrcOps.size() == 1 && "Invalid Srcs");
1322 validateUnaryOp(Res: DstOps[0].getLLTTy(MRI: *getMRI()),
1323 Op0: SrcOps[0].getLLTTy(MRI: *getMRI()));
1324 break;
1325 case TargetOpcode::G_ADD:
1326 case TargetOpcode::G_AND:
1327 case TargetOpcode::G_MUL:
1328 case TargetOpcode::G_OR:
1329 case TargetOpcode::G_SUB:
1330 case TargetOpcode::G_XOR:
1331 case TargetOpcode::G_UDIV:
1332 case TargetOpcode::G_SDIV:
1333 case TargetOpcode::G_UREM:
1334 case TargetOpcode::G_SREM:
1335 case TargetOpcode::G_SMIN:
1336 case TargetOpcode::G_SMAX:
1337 case TargetOpcode::G_UMIN:
1338 case TargetOpcode::G_UMAX:
1339 case TargetOpcode::G_UADDSAT:
1340 case TargetOpcode::G_SADDSAT:
1341 case TargetOpcode::G_USUBSAT:
1342 case TargetOpcode::G_SSUBSAT: {
1343 // All these are binary ops.
1344 assert(DstOps.size() == 1 && "Invalid Dst");
1345 assert(SrcOps.size() == 2 && "Invalid Srcs");
1346 validateBinaryOp(Res: DstOps[0].getLLTTy(MRI: *getMRI()),
1347 Op0: SrcOps[0].getLLTTy(MRI: *getMRI()),
1348 Op1: SrcOps[1].getLLTTy(MRI: *getMRI()));
1349 break;
1350 }
1351 case TargetOpcode::G_SHL:
1352 case TargetOpcode::G_ASHR:
1353 case TargetOpcode::G_LSHR:
1354 case TargetOpcode::G_USHLSAT:
1355 case TargetOpcode::G_SSHLSAT: {
1356 assert(DstOps.size() == 1 && "Invalid Dst");
1357 assert(SrcOps.size() == 2 && "Invalid Srcs");
1358 validateShiftOp(Res: DstOps[0].getLLTTy(MRI: *getMRI()),
1359 Op0: SrcOps[0].getLLTTy(MRI: *getMRI()),
1360 Op1: SrcOps[1].getLLTTy(MRI: *getMRI()));
1361 break;
1362 }
1363 case TargetOpcode::G_SEXT:
1364 case TargetOpcode::G_ZEXT:
1365 case TargetOpcode::G_ANYEXT:
1366 assert(DstOps.size() == 1 && "Invalid Dst");
1367 assert(SrcOps.size() == 1 && "Invalid Srcs");
1368 validateTruncExt(DstTy: DstOps[0].getLLTTy(MRI: *getMRI()),
1369 SrcTy: SrcOps[0].getLLTTy(MRI: *getMRI()), IsExtend: true);
1370 break;
1371 case TargetOpcode::G_TRUNC:
1372 case TargetOpcode::G_FPTRUNC: {
1373 assert(DstOps.size() == 1 && "Invalid Dst");
1374 assert(SrcOps.size() == 1 && "Invalid Srcs");
1375 validateTruncExt(DstTy: DstOps[0].getLLTTy(MRI: *getMRI()),
1376 SrcTy: SrcOps[0].getLLTTy(MRI: *getMRI()), IsExtend: false);
1377 break;
1378 }
1379 case TargetOpcode::G_BITCAST: {
1380 assert(DstOps.size() == 1 && "Invalid Dst");
1381 assert(SrcOps.size() == 1 && "Invalid Srcs");
1382 assert(DstOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1383 SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() && "invalid bitcast");
1384 break;
1385 }
1386 case TargetOpcode::COPY:
1387 assert(DstOps.size() == 1 && "Invalid Dst");
1388 // If the caller wants to add a subreg source it has to be done separately
1389 // so we may not have any SrcOps at this point yet.
1390 break;
1391 case TargetOpcode::G_FCMP:
1392 case TargetOpcode::G_ICMP: {
1393 assert(DstOps.size() == 1 && "Invalid Dst Operands");
1394 assert(SrcOps.size() == 3 && "Invalid Src Operands");
1395 // For F/ICMP, the first src operand is the predicate, followed by
1396 // the two comparands.
1397 assert(SrcOps[0].getSrcOpKind() == SrcOp::SrcType::Ty_Predicate &&
1398 "Expecting predicate");
1399 assert([&]() -> bool {
1400 CmpInst::Predicate Pred = SrcOps[0].getPredicate();
1401 return Opc == TargetOpcode::G_ICMP ? CmpInst::isIntPredicate(Pred)
1402 : CmpInst::isFPPredicate(Pred);
1403 }() && "Invalid predicate");
1404 assert(SrcOps[1].getLLTTy(*getMRI()) == SrcOps[2].getLLTTy(*getMRI()) &&
1405 "Type mismatch");
1406 assert([&]() -> bool {
1407 LLT Op0Ty = SrcOps[1].getLLTTy(*getMRI());
1408 LLT DstTy = DstOps[0].getLLTTy(*getMRI());
1409 if (Op0Ty.isScalar() || Op0Ty.isPointer())
1410 return DstTy.isScalar();
1411 else
1412 return DstTy.isVector() &&
1413 DstTy.getElementCount() == Op0Ty.getElementCount();
1414 }() && "Type Mismatch");
1415 break;
1416 }
1417 case TargetOpcode::G_UNMERGE_VALUES: {
1418 assert(!DstOps.empty() && "Invalid trivial sequence");
1419 assert(SrcOps.size() == 1 && "Invalid src for Unmerge");
1420 assert(llvm::all_of(DstOps,
1421 [&, this](const DstOp &Op) {
1422 return Op.getLLTTy(*getMRI()) ==
1423 DstOps[0].getLLTTy(*getMRI());
1424 }) &&
1425 "type mismatch in output list");
1426 assert((TypeSize::ScalarTy)DstOps.size() *
1427 DstOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1428 SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1429 "input operands do not cover output register");
1430 break;
1431 }
1432 case TargetOpcode::G_MERGE_VALUES: {
1433 assert(SrcOps.size() >= 2 && "invalid trivial sequence");
1434 assert(DstOps.size() == 1 && "Invalid Dst");
1435 assert(llvm::all_of(SrcOps,
1436 [&, this](const SrcOp &Op) {
1437 return Op.getLLTTy(*getMRI()) ==
1438 SrcOps[0].getLLTTy(*getMRI());
1439 }) &&
1440 "type mismatch in input list");
1441 assert((TypeSize::ScalarTy)SrcOps.size() *
1442 SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1443 DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1444 "input operands do not cover output register");
1445 assert(!DstOps[0].getLLTTy(*getMRI()).isVector() &&
1446 "vectors should be built with G_CONCAT_VECTOR or G_BUILD_VECTOR");
1447 break;
1448 }
1449 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
1450 assert(DstOps.size() == 1 && "Invalid Dst size");
1451 assert(SrcOps.size() == 2 && "Invalid Src size");
1452 assert(SrcOps[0].getLLTTy(*getMRI()).isVector() && "Invalid operand type");
1453 assert((DstOps[0].getLLTTy(*getMRI()).isScalar() ||
1454 DstOps[0].getLLTTy(*getMRI()).isPointer()) &&
1455 "Invalid operand type");
1456 assert(SrcOps[1].getLLTTy(*getMRI()).isScalar() && "Invalid operand type");
1457 assert(SrcOps[0].getLLTTy(*getMRI()).getElementType() ==
1458 DstOps[0].getLLTTy(*getMRI()) &&
1459 "Type mismatch");
1460 break;
1461 }
1462 case TargetOpcode::G_INSERT_VECTOR_ELT: {
1463 assert(DstOps.size() == 1 && "Invalid dst size");
1464 assert(SrcOps.size() == 3 && "Invalid src size");
1465 assert(DstOps[0].getLLTTy(*getMRI()).isVector() &&
1466 SrcOps[0].getLLTTy(*getMRI()).isVector() && "Invalid operand type");
1467 assert(DstOps[0].getLLTTy(*getMRI()).getElementType() ==
1468 SrcOps[1].getLLTTy(*getMRI()) &&
1469 "Type mismatch");
1470 assert(SrcOps[2].getLLTTy(*getMRI()).isScalar() && "Invalid index");
1471 assert(DstOps[0].getLLTTy(*getMRI()).getElementCount() ==
1472 SrcOps[0].getLLTTy(*getMRI()).getElementCount() &&
1473 "Type mismatch");
1474 break;
1475 }
1476 case TargetOpcode::G_INSERT_SUBVECTOR: {
1477 assert(DstOps.size() == 1 && "Invalid Dst");
1478 assert(SrcOps.size() == 3 && "Invalid Srcs");
1479 [[maybe_unused]] LLT DstTy = DstOps[0].getLLTTy(MRI: *getMRI());
1480 [[maybe_unused]] LLT BigVecTy = SrcOps[0].getLLTTy(MRI: *getMRI());
1481 [[maybe_unused]] LLT SubVecTy = SrcOps[1].getLLTTy(MRI: *getMRI());
1482 assert(DstTy == BigVecTy &&
1483 "Dest and insert subvector source types must match!");
1484 assert(DstTy.isVector() && SubVecTy.isVector() &&
1485 "Insert subvector VTs must be vectors!");
1486 assert(DstTy.getElementType() == SubVecTy.getElementType() &&
1487 "Insert subvector VTs must have the same element type!");
1488 assert((DstTy.isScalable() || !SubVecTy.isScalable()) &&
1489 "Cannot insert a scalable vector into a fixed length vector!");
1490 assert((DstTy.isScalable() != SubVecTy.isScalable() ||
1491 DstTy.getElementCount().getKnownMinValue() >=
1492 SubVecTy.getElementCount().getKnownMinValue()) &&
1493 "Insert subvector must be from smaller vector to larger vector!");
1494 assert(SrcOps[2].getSrcOpKind() == SrcOp::SrcType::Ty_Imm &&
1495 "Insert subvector index must be constant");
1496 assert((DstTy.isScalable() != SubVecTy.isScalable() ||
1497 (SubVecTy.getElementCount().getKnownMinValue() +
1498 (uint64_t)SrcOps[2].getImm()) <=
1499 DstTy.getElementCount().getKnownMinValue()) &&
1500 "Insert subvector overflow!");
1501 assert((uint64_t)SrcOps[2].getImm() %
1502 SubVecTy.getElementCount().getKnownMinValue() ==
1503 0 &&
1504 "Insert index is not a multiple of the subvector length");
1505 break;
1506 }
1507 case TargetOpcode::G_EXTRACT_SUBVECTOR: {
1508 assert(DstOps.size() == 1 && "Invalid Dst");
1509 assert(SrcOps.size() == 2 && "Invalid Srcs");
1510 [[maybe_unused]] LLT DstTy = DstOps[0].getLLTTy(MRI: *getMRI());
1511 [[maybe_unused]] LLT SrcVecTy = SrcOps[0].getLLTTy(MRI: *getMRI());
1512 assert(DstTy.isVector() && SrcVecTy.isVector() &&
1513 "Extract subvector VTs must be vectors!");
1514 assert(DstTy.getElementType() == SrcVecTy.getElementType() &&
1515 "Extract subvector VTs must have the same element type!");
1516 assert((!DstTy.isScalable() || SrcVecTy.isScalable()) &&
1517 "Cannot extract a scalable vector from a fixed length vector!");
1518 assert((DstTy.isScalable() != SrcVecTy.isScalable() ||
1519 DstTy.getElementCount().getKnownMinValue() <=
1520 SrcVecTy.getElementCount().getKnownMinValue()) &&
1521 "Extract subvector must be from larger vector to smaller vector!");
1522 assert(SrcOps[1].getSrcOpKind() == SrcOp::SrcType::Ty_Imm &&
1523 "Extract subvector index must be a constant");
1524 assert((DstTy.isScalable() != SrcVecTy.isScalable() ||
1525 (DstTy.getElementCount().getKnownMinValue() +
1526 (uint64_t)SrcOps[1].getImm()) <=
1527 SrcVecTy.getElementCount().getKnownMinValue()) &&
1528 "Extract subvector overflow!");
1529 assert((uint64_t)SrcOps[1].getImm() %
1530 DstTy.getElementCount().getKnownMinValue() ==
1531 0 &&
1532 "Extract index is not a multiple of the output vector length");
1533 break;
1534 }
1535 case TargetOpcode::G_BUILD_VECTOR: {
1536 assert((!SrcOps.empty() || SrcOps.size() < 2) &&
1537 "Must have at least 2 operands");
1538 assert(DstOps.size() == 1 && "Invalid DstOps");
1539 assert(DstOps[0].getLLTTy(*getMRI()).isVector() &&
1540 "Res type must be a vector");
1541 assert(llvm::all_of(SrcOps,
1542 [&, this](const SrcOp &Op) {
1543 return Op.getLLTTy(*getMRI()) ==
1544 SrcOps[0].getLLTTy(*getMRI());
1545 }) &&
1546 "type mismatch in input list");
1547 assert((TypeSize::ScalarTy)SrcOps.size() *
1548 SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1549 DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1550 "input scalars do not exactly cover the output vector register");
1551 break;
1552 }
1553 case TargetOpcode::G_BUILD_VECTOR_TRUNC: {
1554 assert((!SrcOps.empty() || SrcOps.size() < 2) &&
1555 "Must have at least 2 operands");
1556 assert(DstOps.size() == 1 && "Invalid DstOps");
1557 assert(DstOps[0].getLLTTy(*getMRI()).isVector() &&
1558 "Res type must be a vector");
1559 assert(llvm::all_of(SrcOps,
1560 [&, this](const SrcOp &Op) {
1561 return Op.getLLTTy(*getMRI()) ==
1562 SrcOps[0].getLLTTy(*getMRI());
1563 }) &&
1564 "type mismatch in input list");
1565 break;
1566 }
1567 case TargetOpcode::G_CONCAT_VECTORS: {
1568 assert(DstOps.size() == 1 && "Invalid DstOps");
1569 assert((!SrcOps.empty() || SrcOps.size() < 2) &&
1570 "Must have at least 2 operands");
1571 assert(llvm::all_of(SrcOps,
1572 [&, this](const SrcOp &Op) {
1573 return (Op.getLLTTy(*getMRI()).isVector() &&
1574 Op.getLLTTy(*getMRI()) ==
1575 SrcOps[0].getLLTTy(*getMRI()));
1576 }) &&
1577 "type mismatch in input list");
1578 assert((TypeSize::ScalarTy)SrcOps.size() *
1579 SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1580 DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1581 "input vectors do not exactly cover the output vector register");
1582 break;
1583 }
1584 case TargetOpcode::G_UADDE: {
1585 assert(DstOps.size() == 2 && "Invalid no of dst operands");
1586 assert(SrcOps.size() == 3 && "Invalid no of src operands");
1587 assert(DstOps[0].getLLTTy(*getMRI()).isScalar() && "Invalid operand");
1588 assert((DstOps[0].getLLTTy(*getMRI()) == SrcOps[0].getLLTTy(*getMRI())) &&
1589 (DstOps[0].getLLTTy(*getMRI()) == SrcOps[1].getLLTTy(*getMRI())) &&
1590 "Invalid operand");
1591 assert(DstOps[1].getLLTTy(*getMRI()).isScalar() && "Invalid operand");
1592 assert(DstOps[1].getLLTTy(*getMRI()) == SrcOps[2].getLLTTy(*getMRI()) &&
1593 "type mismatch");
1594 break;
1595 }
1596 }
1597
1598 auto MIB = buildInstr(Opcode: Opc);
1599 for (const DstOp &Op : DstOps)
1600 Op.addDefToMIB(MRI&: *getMRI(), MIB);
1601 for (const SrcOp &Op : SrcOps)
1602 Op.addSrcToMIB(MIB);
1603 if (Flags)
1604 MIB->setFlags(*Flags);
1605 return MIB;
1606}
1607