1//===-- llvm/CodeGen/GlobalISel/MachineIRBuilder.cpp - MIBuilder--*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the MachineIRBuidler class.
10//===----------------------------------------------------------------------===//
11#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
12#include "llvm/CodeGen/MachineFunction.h"
13#include "llvm/CodeGen/MachineInstr.h"
14#include "llvm/CodeGen/MachineInstrBuilder.h"
15#include "llvm/CodeGen/MachineRegisterInfo.h"
16#include "llvm/CodeGen/TargetInstrInfo.h"
17#include "llvm/CodeGen/TargetLowering.h"
18#include "llvm/CodeGen/TargetOpcodes.h"
19#include "llvm/CodeGen/TargetSubtargetInfo.h"
20#include "llvm/IR/DebugInfoMetadata.h"
21
22using namespace llvm;
23
24void MachineIRBuilder::setMF(MachineFunction &MF) {
25 State.MF = &MF;
26 State.MBB = nullptr;
27 State.MRI = &MF.getRegInfo();
28 State.TII = MF.getSubtarget().getInstrInfo();
29 State.DL = DebugLoc();
30 State.PCSections = nullptr;
31 State.MMRA = nullptr;
32 State.II = MachineBasicBlock::iterator();
33 State.Observer = nullptr;
34}
35
36//------------------------------------------------------------------------------
37// Build instruction variants.
38//------------------------------------------------------------------------------
39
40MachineInstrBuilder MachineIRBuilder::buildInstrNoInsert(unsigned Opcode) {
41 return BuildMI(MF&: getMF(), MIMD: {getDL(), getPCSections(), getMMRAMetadata()},
42 MCID: getTII().get(Opcode));
43}
44
45MachineInstrBuilder MachineIRBuilder::insertInstr(MachineInstrBuilder MIB) {
46 getMBB().insert(I: getInsertPt(), MI: MIB);
47 recordInsertion(InsertedInstr: MIB);
48 return MIB;
49}
50
51MachineInstrBuilder
52MachineIRBuilder::buildDirectDbgValue(Register Reg, const MDNode *Variable,
53 const MDNode *Expr) {
54 assert(isa<DILocalVariable>(Variable) && "not a variable");
55 assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
56 assert(
57 cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
58 "Expected inlined-at fields to agree");
59 return insertInstr(MIB: BuildMI(MF&: getMF(), DL: getDL(),
60 MCID: getTII().get(Opcode: TargetOpcode::DBG_VALUE),
61 /*IsIndirect*/ false, Reg, Variable, Expr));
62}
63
64MachineInstrBuilder
65MachineIRBuilder::buildIndirectDbgValue(Register Reg, const MDNode *Variable,
66 const MDNode *Expr) {
67 assert(isa<DILocalVariable>(Variable) && "not a variable");
68 assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
69 assert(
70 cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
71 "Expected inlined-at fields to agree");
72 return insertInstr(MIB: BuildMI(MF&: getMF(), DL: getDL(),
73 MCID: getTII().get(Opcode: TargetOpcode::DBG_VALUE),
74 /*IsIndirect*/ true, Reg, Variable, Expr));
75}
76
77MachineInstrBuilder MachineIRBuilder::buildFIDbgValue(int FI,
78 const MDNode *Variable,
79 const MDNode *Expr) {
80 assert(isa<DILocalVariable>(Variable) && "not a variable");
81 assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
82 assert(
83 cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
84 "Expected inlined-at fields to agree");
85 return insertInstr(MIB: buildInstrNoInsert(Opcode: TargetOpcode::DBG_VALUE)
86 .addFrameIndex(Idx: FI)
87 .addImm(Val: 0)
88 .addMetadata(MD: Variable)
89 .addMetadata(MD: Expr));
90}
91
92MachineInstrBuilder MachineIRBuilder::buildConstDbgValue(const Constant &C,
93 const MDNode *Variable,
94 const MDNode *Expr) {
95 assert(isa<DILocalVariable>(Variable) && "not a variable");
96 assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
97 assert(
98 cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
99 "Expected inlined-at fields to agree");
100 auto MIB = buildInstrNoInsert(Opcode: TargetOpcode::DBG_VALUE);
101
102 auto *NumericConstant = [&] () -> const Constant* {
103 if (const auto *CE = dyn_cast<ConstantExpr>(Val: &C))
104 if (CE->getOpcode() == Instruction::IntToPtr)
105 return CE->getOperand(i_nocapture: 0);
106 return &C;
107 }();
108
109 if (auto *CI = dyn_cast<ConstantInt>(Val: NumericConstant)) {
110 if (CI->getBitWidth() > 64)
111 MIB.addCImm(Val: CI);
112 else
113 MIB.addImm(Val: CI->getZExtValue());
114 } else if (auto *CFP = dyn_cast<ConstantFP>(Val: NumericConstant)) {
115 MIB.addFPImm(Val: CFP);
116 } else if (isa<ConstantPointerNull>(Val: NumericConstant)) {
117 MIB.addImm(Val: 0);
118 } else {
119 // Insert $noreg if we didn't find a usable constant and had to drop it.
120 MIB.addReg(RegNo: Register());
121 }
122
123 MIB.addImm(Val: 0).addMetadata(MD: Variable).addMetadata(MD: Expr);
124 return insertInstr(MIB);
125}
126
127MachineInstrBuilder MachineIRBuilder::buildDbgLabel(const MDNode *Label) {
128 assert(isa<DILabel>(Label) && "not a label");
129 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(State.DL) &&
130 "Expected inlined-at fields to agree");
131 auto MIB = buildInstr(Opcode: TargetOpcode::DBG_LABEL);
132
133 return MIB.addMetadata(MD: Label);
134}
135
136MachineInstrBuilder MachineIRBuilder::buildDynStackAlloc(const DstOp &Res,
137 const SrcOp &Size,
138 Align Alignment) {
139 assert(Res.getLLTTy(*getMRI()).isPointer() && "expected ptr dst type");
140 auto MIB = buildInstr(Opcode: TargetOpcode::G_DYN_STACKALLOC);
141 Res.addDefToMIB(MRI&: *getMRI(), MIB);
142 Size.addSrcToMIB(MIB);
143 MIB.addImm(Val: Alignment.value());
144 return MIB;
145}
146
147MachineInstrBuilder MachineIRBuilder::buildFrameIndex(const DstOp &Res,
148 int Idx) {
149 assert(Res.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
150 auto MIB = buildInstr(Opcode: TargetOpcode::G_FRAME_INDEX);
151 Res.addDefToMIB(MRI&: *getMRI(), MIB);
152 MIB.addFrameIndex(Idx);
153 return MIB;
154}
155
156MachineInstrBuilder MachineIRBuilder::buildGlobalValue(const DstOp &Res,
157 const GlobalValue *GV) {
158 assert(Res.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
159 assert(Res.getLLTTy(*getMRI()).getAddressSpace() ==
160 GV->getType()->getAddressSpace() &&
161 "address space mismatch");
162
163 auto MIB = buildInstr(Opcode: TargetOpcode::G_GLOBAL_VALUE);
164 Res.addDefToMIB(MRI&: *getMRI(), MIB);
165 MIB.addGlobalAddress(GV);
166 return MIB;
167}
168
169MachineInstrBuilder MachineIRBuilder::buildConstantPool(const DstOp &Res,
170 unsigned Idx) {
171 assert(Res.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
172 auto MIB = buildInstr(Opcode: TargetOpcode::G_CONSTANT_POOL);
173 Res.addDefToMIB(MRI&: *getMRI(), MIB);
174 MIB.addConstantPoolIndex(Idx);
175 return MIB;
176}
177
178MachineInstrBuilder MachineIRBuilder::buildJumpTable(const LLT PtrTy,
179 unsigned JTI) {
180 return buildInstr(Opc: TargetOpcode::G_JUMP_TABLE, DstOps: {PtrTy}, SrcOps: {})
181 .addJumpTableIndex(Idx: JTI);
182}
183
184void MachineIRBuilder::validateUnaryOp(const LLT Res, const LLT Op0) {
185 assert((Res.isScalar() || Res.isVector()) && "invalid operand type");
186 assert((Res == Op0) && "type mismatch");
187}
188
189void MachineIRBuilder::validateBinaryOp(const LLT Res, const LLT Op0,
190 const LLT Op1) {
191 assert((Res.isScalar() || Res.isVector()) && "invalid operand type");
192 assert((Res == Op0 && Res == Op1) && "type mismatch");
193}
194
195void MachineIRBuilder::validateShiftOp(const LLT Res, const LLT Op0,
196 const LLT Op1) {
197 assert((Res.isScalar() || Res.isVector()) && "invalid operand type");
198 assert((Res == Op0) && "type mismatch");
199}
200
201MachineInstrBuilder
202MachineIRBuilder::buildPtrAdd(const DstOp &Res, const SrcOp &Op0,
203 const SrcOp &Op1, std::optional<unsigned> Flags) {
204 assert(Res.getLLTTy(*getMRI()).isPointerOrPointerVector() &&
205 Res.getLLTTy(*getMRI()) == Op0.getLLTTy(*getMRI()) && "type mismatch");
206 assert(Op1.getLLTTy(*getMRI()).getScalarType().isScalar() && "invalid offset type");
207
208 return buildInstr(Opc: TargetOpcode::G_PTR_ADD, DstOps: {Res}, SrcOps: {Op0, Op1}, Flags);
209}
210
211std::optional<MachineInstrBuilder>
212MachineIRBuilder::materializePtrAdd(Register &Res, Register Op0,
213 const LLT ValueTy, uint64_t Value) {
214 assert(Res == 0 && "Res is a result argument");
215 assert(ValueTy.isScalar() && "invalid offset type");
216
217 if (Value == 0) {
218 Res = Op0;
219 return std::nullopt;
220 }
221
222 Res = getMRI()->createGenericVirtualRegister(Ty: getMRI()->getType(Reg: Op0));
223 auto Cst = buildConstant(Res: ValueTy, Val: Value);
224 return buildPtrAdd(Res, Op0, Op1: Cst.getReg(Idx: 0));
225}
226
227MachineInstrBuilder MachineIRBuilder::buildMaskLowPtrBits(const DstOp &Res,
228 const SrcOp &Op0,
229 uint32_t NumBits) {
230 LLT PtrTy = Res.getLLTTy(MRI: *getMRI());
231 LLT MaskTy = LLT::scalar(SizeInBits: PtrTy.getSizeInBits());
232 Register MaskReg = getMRI()->createGenericVirtualRegister(Ty: MaskTy);
233 buildConstant(Res: MaskReg, Val: maskTrailingZeros<uint64_t>(N: NumBits));
234 return buildPtrMask(Res, Op0, Op1: MaskReg);
235}
236
237MachineInstrBuilder
238MachineIRBuilder::buildPadVectorWithUndefElements(const DstOp &Res,
239 const SrcOp &Op0) {
240 LLT ResTy = Res.getLLTTy(MRI: *getMRI());
241 LLT Op0Ty = Op0.getLLTTy(MRI: *getMRI());
242
243 assert(ResTy.isVector() && "Res non vector type");
244
245 SmallVector<Register, 8> Regs;
246 if (Op0Ty.isVector()) {
247 assert((ResTy.getElementType() == Op0Ty.getElementType()) &&
248 "Different vector element types");
249 assert((ResTy.getNumElements() > Op0Ty.getNumElements()) &&
250 "Op0 has more elements");
251 auto Unmerge = buildUnmerge(Res: Op0Ty.getElementType(), Op: Op0);
252
253 for (auto Op : Unmerge.getInstr()->defs())
254 Regs.push_back(Elt: Op.getReg());
255 } else {
256 assert((ResTy.getSizeInBits() > Op0Ty.getSizeInBits()) &&
257 "Op0 has more size");
258 Regs.push_back(Elt: Op0.getReg());
259 }
260 Register Undef =
261 buildUndef(Res: Op0Ty.isVector() ? Op0Ty.getElementType() : Op0Ty).getReg(Idx: 0);
262 unsigned NumberOfPadElts = ResTy.getNumElements() - Regs.size();
263 for (unsigned i = 0; i < NumberOfPadElts; ++i)
264 Regs.push_back(Elt: Undef);
265 return buildMergeLikeInstr(Res, Ops: Regs);
266}
267
268MachineInstrBuilder
269MachineIRBuilder::buildDeleteTrailingVectorElements(const DstOp &Res,
270 const SrcOp &Op0) {
271 LLT ResTy = Res.getLLTTy(MRI: *getMRI());
272 LLT Op0Ty = Op0.getLLTTy(MRI: *getMRI());
273
274 assert(Op0Ty.isVector() && "Non vector type");
275 assert(((ResTy.isScalar() && (ResTy == Op0Ty.getElementType())) ||
276 (ResTy.isVector() &&
277 (ResTy.getElementType() == Op0Ty.getElementType()))) &&
278 "Different vector element types");
279 assert(
280 (ResTy.isScalar() || (ResTy.getNumElements() < Op0Ty.getNumElements())) &&
281 "Op0 has fewer elements");
282
283 auto Unmerge = buildUnmerge(Res: Op0Ty.getElementType(), Op: Op0);
284 if (ResTy.isScalar())
285 return buildCopy(Res, Op: Unmerge.getReg(Idx: 0));
286 SmallVector<Register, 8> Regs;
287 for (unsigned i = 0; i < ResTy.getNumElements(); ++i)
288 Regs.push_back(Elt: Unmerge.getReg(Idx: i));
289 return buildMergeLikeInstr(Res, Ops: Regs);
290}
291
292MachineInstrBuilder MachineIRBuilder::buildBr(MachineBasicBlock &Dest) {
293 return buildInstr(Opcode: TargetOpcode::G_BR).addMBB(MBB: &Dest);
294}
295
296MachineInstrBuilder MachineIRBuilder::buildBrIndirect(Register Tgt) {
297 assert(getMRI()->getType(Tgt).isPointer() && "invalid branch destination");
298 return buildInstr(Opcode: TargetOpcode::G_BRINDIRECT).addUse(RegNo: Tgt);
299}
300
301MachineInstrBuilder MachineIRBuilder::buildBrJT(Register TablePtr,
302 unsigned JTI,
303 Register IndexReg) {
304 assert(getMRI()->getType(TablePtr).isPointer() &&
305 "Table reg must be a pointer");
306 return buildInstr(Opcode: TargetOpcode::G_BRJT)
307 .addUse(RegNo: TablePtr)
308 .addJumpTableIndex(Idx: JTI)
309 .addUse(RegNo: IndexReg);
310}
311
312MachineInstrBuilder MachineIRBuilder::buildCopy(const DstOp &Res,
313 const SrcOp &Op) {
314 return buildInstr(Opc: TargetOpcode::COPY, DstOps: Res, SrcOps: Op);
315}
316
317MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
318 const ConstantInt &Val) {
319 LLT Ty = Res.getLLTTy(MRI: *getMRI());
320 LLT EltTy = Ty.getScalarType();
321 assert(EltTy.getScalarSizeInBits() == Val.getBitWidth() &&
322 "creating constant with the wrong size");
323
324 assert(!Ty.isScalableVector() &&
325 "unexpected scalable vector in buildConstant");
326
327 if (Ty.isFixedVector()) {
328 auto Const = buildInstr(Opcode: TargetOpcode::G_CONSTANT)
329 .addDef(RegNo: getMRI()->createGenericVirtualRegister(Ty: EltTy))
330 .addCImm(Val: &Val);
331 return buildSplatBuildVector(Res, Src: Const);
332 }
333
334 auto Const = buildInstr(Opcode: TargetOpcode::G_CONSTANT);
335 Const->setDebugLoc(DebugLoc());
336 Res.addDefToMIB(MRI&: *getMRI(), MIB&: Const);
337 Const.addCImm(Val: &Val);
338 return Const;
339}
340
341MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
342 int64_t Val) {
343 auto IntN = IntegerType::get(C&: getMF().getFunction().getContext(),
344 NumBits: Res.getLLTTy(MRI: *getMRI()).getScalarSizeInBits());
345 ConstantInt *CI = ConstantInt::get(Ty: IntN, V: Val, IsSigned: true);
346 return buildConstant(Res, Val: *CI);
347}
348
349MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
350 const ConstantFP &Val) {
351 LLT Ty = Res.getLLTTy(MRI: *getMRI());
352 LLT EltTy = Ty.getScalarType();
353
354 assert(APFloat::getSizeInBits(Val.getValueAPF().getSemantics())
355 == EltTy.getSizeInBits() &&
356 "creating fconstant with the wrong size");
357
358 assert(!Ty.isPointer() && "invalid operand type");
359
360 assert(!Ty.isScalableVector() &&
361 "unexpected scalable vector in buildFConstant");
362
363 if (Ty.isFixedVector()) {
364 auto Const = buildInstr(Opcode: TargetOpcode::G_FCONSTANT)
365 .addDef(RegNo: getMRI()->createGenericVirtualRegister(Ty: EltTy))
366 .addFPImm(Val: &Val);
367
368 return buildSplatBuildVector(Res, Src: Const);
369 }
370
371 auto Const = buildInstr(Opcode: TargetOpcode::G_FCONSTANT);
372 Const->setDebugLoc(DebugLoc());
373 Res.addDefToMIB(MRI&: *getMRI(), MIB&: Const);
374 Const.addFPImm(Val: &Val);
375 return Const;
376}
377
378MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
379 const APInt &Val) {
380 ConstantInt *CI = ConstantInt::get(Context&: getMF().getFunction().getContext(), V: Val);
381 return buildConstant(Res, Val: *CI);
382}
383
384MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
385 double Val) {
386 LLT DstTy = Res.getLLTTy(MRI: *getMRI());
387 auto &Ctx = getMF().getFunction().getContext();
388 auto *CFP =
389 ConstantFP::get(Context&: Ctx, V: getAPFloatFromSize(Val, Size: DstTy.getScalarSizeInBits()));
390 return buildFConstant(Res, Val: *CFP);
391}
392
393MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
394 const APFloat &Val) {
395 auto &Ctx = getMF().getFunction().getContext();
396 auto *CFP = ConstantFP::get(Context&: Ctx, V: Val);
397 return buildFConstant(Res, Val: *CFP);
398}
399
400MachineInstrBuilder
401MachineIRBuilder::buildConstantPtrAuth(const DstOp &Res,
402 const ConstantPtrAuth *CPA,
403 Register Addr, Register AddrDisc) {
404 auto MIB = buildInstr(Opcode: TargetOpcode::G_PTRAUTH_GLOBAL_VALUE);
405 Res.addDefToMIB(MRI&: *getMRI(), MIB);
406 MIB.addUse(RegNo: Addr);
407 MIB.addImm(Val: CPA->getKey()->getZExtValue());
408 MIB.addUse(RegNo: AddrDisc);
409 MIB.addImm(Val: CPA->getDiscriminator()->getZExtValue());
410 return MIB;
411}
412
413MachineInstrBuilder MachineIRBuilder::buildBrCond(const SrcOp &Tst,
414 MachineBasicBlock &Dest) {
415 assert(Tst.getLLTTy(*getMRI()).isScalar() && "invalid operand type");
416
417 auto MIB = buildInstr(Opcode: TargetOpcode::G_BRCOND);
418 Tst.addSrcToMIB(MIB);
419 MIB.addMBB(MBB: &Dest);
420 return MIB;
421}
422
423MachineInstrBuilder
424MachineIRBuilder::buildLoad(const DstOp &Dst, const SrcOp &Addr,
425 MachinePointerInfo PtrInfo, Align Alignment,
426 MachineMemOperand::Flags MMOFlags,
427 const AAMDNodes &AAInfo) {
428 MMOFlags |= MachineMemOperand::MOLoad;
429 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
430
431 LLT Ty = Dst.getLLTTy(MRI: *getMRI());
432 MachineMemOperand *MMO =
433 getMF().getMachineMemOperand(PtrInfo, f: MMOFlags, MemTy: Ty, base_alignment: Alignment, AAInfo);
434 return buildLoad(Res: Dst, Addr, MMO&: *MMO);
435}
436
437MachineInstrBuilder MachineIRBuilder::buildLoadInstr(unsigned Opcode,
438 const DstOp &Res,
439 const SrcOp &Addr,
440 MachineMemOperand &MMO) {
441 assert(Res.getLLTTy(*getMRI()).isValid() && "invalid operand type");
442 assert(Addr.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
443
444 auto MIB = buildInstr(Opcode);
445 Res.addDefToMIB(MRI&: *getMRI(), MIB);
446 Addr.addSrcToMIB(MIB);
447 MIB.addMemOperand(MMO: &MMO);
448 return MIB;
449}
450
451MachineInstrBuilder MachineIRBuilder::buildLoadFromOffset(
452 const DstOp &Dst, const SrcOp &BasePtr,
453 MachineMemOperand &BaseMMO, int64_t Offset) {
454 LLT LoadTy = Dst.getLLTTy(MRI: *getMRI());
455 MachineMemOperand *OffsetMMO =
456 getMF().getMachineMemOperand(MMO: &BaseMMO, Offset, Ty: LoadTy);
457
458 if (Offset == 0) // This may be a size or type changing load.
459 return buildLoad(Res: Dst, Addr: BasePtr, MMO&: *OffsetMMO);
460
461 LLT PtrTy = BasePtr.getLLTTy(MRI: *getMRI());
462 LLT OffsetTy = LLT::scalar(SizeInBits: PtrTy.getSizeInBits());
463 auto ConstOffset = buildConstant(Res: OffsetTy, Val: Offset);
464 auto Ptr = buildPtrAdd(Res: PtrTy, Op0: BasePtr, Op1: ConstOffset);
465 return buildLoad(Res: Dst, Addr: Ptr, MMO&: *OffsetMMO);
466}
467
468MachineInstrBuilder MachineIRBuilder::buildStore(const SrcOp &Val,
469 const SrcOp &Addr,
470 MachineMemOperand &MMO) {
471 assert(Val.getLLTTy(*getMRI()).isValid() && "invalid operand type");
472 assert(Addr.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
473
474 auto MIB = buildInstr(Opcode: TargetOpcode::G_STORE);
475 Val.addSrcToMIB(MIB);
476 Addr.addSrcToMIB(MIB);
477 MIB.addMemOperand(MMO: &MMO);
478 return MIB;
479}
480
481MachineInstrBuilder
482MachineIRBuilder::buildStore(const SrcOp &Val, const SrcOp &Addr,
483 MachinePointerInfo PtrInfo, Align Alignment,
484 MachineMemOperand::Flags MMOFlags,
485 const AAMDNodes &AAInfo) {
486 MMOFlags |= MachineMemOperand::MOStore;
487 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
488
489 LLT Ty = Val.getLLTTy(MRI: *getMRI());
490 MachineMemOperand *MMO =
491 getMF().getMachineMemOperand(PtrInfo, f: MMOFlags, MemTy: Ty, base_alignment: Alignment, AAInfo);
492 return buildStore(Val, Addr, MMO&: *MMO);
493}
494
495MachineInstrBuilder MachineIRBuilder::buildAnyExt(const DstOp &Res,
496 const SrcOp &Op) {
497 return buildInstr(Opc: TargetOpcode::G_ANYEXT, DstOps: Res, SrcOps: Op);
498}
499
500MachineInstrBuilder MachineIRBuilder::buildSExt(const DstOp &Res,
501 const SrcOp &Op) {
502 return buildInstr(Opc: TargetOpcode::G_SEXT, DstOps: Res, SrcOps: Op);
503}
504
505MachineInstrBuilder MachineIRBuilder::buildZExt(const DstOp &Res,
506 const SrcOp &Op,
507 std::optional<unsigned> Flags) {
508 return buildInstr(Opc: TargetOpcode::G_ZEXT, DstOps: Res, SrcOps: Op, Flags);
509}
510
511unsigned MachineIRBuilder::getBoolExtOp(bool IsVec, bool IsFP) const {
512 const auto *TLI = getMF().getSubtarget().getTargetLowering();
513 switch (TLI->getBooleanContents(isVec: IsVec, isFloat: IsFP)) {
514 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
515 return TargetOpcode::G_SEXT;
516 case TargetLoweringBase::ZeroOrOneBooleanContent:
517 return TargetOpcode::G_ZEXT;
518 default:
519 return TargetOpcode::G_ANYEXT;
520 }
521}
522
523MachineInstrBuilder MachineIRBuilder::buildBoolExt(const DstOp &Res,
524 const SrcOp &Op,
525 bool IsFP) {
526 unsigned ExtOp = getBoolExtOp(IsVec: getMRI()->getType(Reg: Op.getReg()).isVector(), IsFP);
527 return buildInstr(Opc: ExtOp, DstOps: Res, SrcOps: Op);
528}
529
530MachineInstrBuilder MachineIRBuilder::buildBoolExtInReg(const DstOp &Res,
531 const SrcOp &Op,
532 bool IsVector,
533 bool IsFP) {
534 const auto *TLI = getMF().getSubtarget().getTargetLowering();
535 switch (TLI->getBooleanContents(isVec: IsVector, isFloat: IsFP)) {
536 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
537 return buildSExtInReg(Res, Op, ImmOp: 1);
538 case TargetLoweringBase::ZeroOrOneBooleanContent:
539 return buildZExtInReg(Res, Op, ImmOp: 1);
540 case TargetLoweringBase::UndefinedBooleanContent:
541 return buildCopy(Res, Op);
542 }
543
544 llvm_unreachable("unexpected BooleanContent");
545}
546
547MachineInstrBuilder MachineIRBuilder::buildExtOrTrunc(unsigned ExtOpc,
548 const DstOp &Res,
549 const SrcOp &Op) {
550 assert((TargetOpcode::G_ANYEXT == ExtOpc || TargetOpcode::G_ZEXT == ExtOpc ||
551 TargetOpcode::G_SEXT == ExtOpc) &&
552 "Expecting Extending Opc");
553 assert(Res.getLLTTy(*getMRI()).isScalar() ||
554 Res.getLLTTy(*getMRI()).isVector());
555 assert(Res.getLLTTy(*getMRI()).isScalar() ==
556 Op.getLLTTy(*getMRI()).isScalar());
557
558 unsigned Opcode = TargetOpcode::COPY;
559 if (Res.getLLTTy(MRI: *getMRI()).getSizeInBits() >
560 Op.getLLTTy(MRI: *getMRI()).getSizeInBits())
561 Opcode = ExtOpc;
562 else if (Res.getLLTTy(MRI: *getMRI()).getSizeInBits() <
563 Op.getLLTTy(MRI: *getMRI()).getSizeInBits())
564 Opcode = TargetOpcode::G_TRUNC;
565 else
566 assert(Res.getLLTTy(*getMRI()) == Op.getLLTTy(*getMRI()));
567
568 return buildInstr(Opc: Opcode, DstOps: Res, SrcOps: Op);
569}
570
571MachineInstrBuilder MachineIRBuilder::buildSExtOrTrunc(const DstOp &Res,
572 const SrcOp &Op) {
573 return buildExtOrTrunc(ExtOpc: TargetOpcode::G_SEXT, Res, Op);
574}
575
576MachineInstrBuilder MachineIRBuilder::buildZExtOrTrunc(const DstOp &Res,
577 const SrcOp &Op) {
578 return buildExtOrTrunc(ExtOpc: TargetOpcode::G_ZEXT, Res, Op);
579}
580
581MachineInstrBuilder MachineIRBuilder::buildAnyExtOrTrunc(const DstOp &Res,
582 const SrcOp &Op) {
583 return buildExtOrTrunc(ExtOpc: TargetOpcode::G_ANYEXT, Res, Op);
584}
585
586MachineInstrBuilder MachineIRBuilder::buildZExtInReg(const DstOp &Res,
587 const SrcOp &Op,
588 int64_t ImmOp) {
589 LLT ResTy = Res.getLLTTy(MRI: *getMRI());
590 auto Mask = buildConstant(
591 Res: ResTy, Val: APInt::getLowBitsSet(numBits: ResTy.getScalarSizeInBits(), loBitsSet: ImmOp));
592 return buildAnd(Dst: Res, Src0: Op, Src1: Mask);
593}
594
595MachineInstrBuilder MachineIRBuilder::buildCast(const DstOp &Dst,
596 const SrcOp &Src) {
597 LLT SrcTy = Src.getLLTTy(MRI: *getMRI());
598 LLT DstTy = Dst.getLLTTy(MRI: *getMRI());
599 if (SrcTy == DstTy)
600 return buildCopy(Res: Dst, Op: Src);
601
602 unsigned Opcode;
603 if (SrcTy.isPointer() && DstTy.isScalar())
604 Opcode = TargetOpcode::G_PTRTOINT;
605 else if (DstTy.isPointer() && SrcTy.isScalar())
606 Opcode = TargetOpcode::G_INTTOPTR;
607 else {
608 assert(!SrcTy.isPointer() && !DstTy.isPointer() && "n G_ADDRCAST yet");
609 Opcode = TargetOpcode::G_BITCAST;
610 }
611
612 return buildInstr(Opc: Opcode, DstOps: Dst, SrcOps: Src);
613}
614
615MachineInstrBuilder MachineIRBuilder::buildExtract(const DstOp &Dst,
616 const SrcOp &Src,
617 uint64_t Index) {
618 LLT SrcTy = Src.getLLTTy(MRI: *getMRI());
619 LLT DstTy = Dst.getLLTTy(MRI: *getMRI());
620
621#ifndef NDEBUG
622 assert(SrcTy.isValid() && "invalid operand type");
623 assert(DstTy.isValid() && "invalid operand type");
624 assert(Index + DstTy.getSizeInBits() <= SrcTy.getSizeInBits() &&
625 "extracting off end of register");
626#endif
627
628 if (DstTy.getSizeInBits() == SrcTy.getSizeInBits()) {
629 assert(Index == 0 && "insertion past the end of a register");
630 return buildCast(Dst, Src);
631 }
632
633 auto Extract = buildInstr(Opcode: TargetOpcode::G_EXTRACT);
634 Dst.addDefToMIB(MRI&: *getMRI(), MIB&: Extract);
635 Src.addSrcToMIB(MIB&: Extract);
636 Extract.addImm(Val: Index);
637 return Extract;
638}
639
640MachineInstrBuilder MachineIRBuilder::buildUndef(const DstOp &Res) {
641 return buildInstr(Opc: TargetOpcode::G_IMPLICIT_DEF, DstOps: {Res}, SrcOps: {});
642}
643
644MachineInstrBuilder MachineIRBuilder::buildMergeValues(const DstOp &Res,
645 ArrayRef<Register> Ops) {
646 // Unfortunately to convert from ArrayRef<LLT> to ArrayRef<SrcOp>,
647 // we need some temporary storage for the DstOp objects. Here we use a
648 // sufficiently large SmallVector to not go through the heap.
649 SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end());
650 assert(TmpVec.size() > 1);
651 return buildInstr(Opc: TargetOpcode::G_MERGE_VALUES, DstOps: Res, SrcOps: TmpVec);
652}
653
654MachineInstrBuilder
655MachineIRBuilder::buildMergeLikeInstr(const DstOp &Res,
656 ArrayRef<Register> Ops) {
657 // Unfortunately to convert from ArrayRef<LLT> to ArrayRef<SrcOp>,
658 // we need some temporary storage for the DstOp objects. Here we use a
659 // sufficiently large SmallVector to not go through the heap.
660 SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end());
661 assert(TmpVec.size() > 1);
662 return buildInstr(Opc: getOpcodeForMerge(DstOp: Res, SrcOps: TmpVec), DstOps: Res, SrcOps: TmpVec);
663}
664
665MachineInstrBuilder
666MachineIRBuilder::buildMergeLikeInstr(const DstOp &Res,
667 std::initializer_list<SrcOp> Ops) {
668 assert(Ops.size() > 1);
669 return buildInstr(Opc: getOpcodeForMerge(DstOp: Res, SrcOps: Ops), DstOps: Res, SrcOps: Ops);
670}
671
672unsigned MachineIRBuilder::getOpcodeForMerge(const DstOp &DstOp,
673 ArrayRef<SrcOp> SrcOps) const {
674 if (DstOp.getLLTTy(MRI: *getMRI()).isVector()) {
675 if (SrcOps[0].getLLTTy(MRI: *getMRI()).isVector())
676 return TargetOpcode::G_CONCAT_VECTORS;
677 return TargetOpcode::G_BUILD_VECTOR;
678 }
679
680 return TargetOpcode::G_MERGE_VALUES;
681}
682
683MachineInstrBuilder MachineIRBuilder::buildUnmerge(ArrayRef<LLT> Res,
684 const SrcOp &Op) {
685 // Unfortunately to convert from ArrayRef<LLT> to ArrayRef<DstOp>,
686 // we need some temporary storage for the DstOp objects. Here we use a
687 // sufficiently large SmallVector to not go through the heap.
688 SmallVector<DstOp, 8> TmpVec(Res.begin(), Res.end());
689 assert(TmpVec.size() > 1);
690 return buildInstr(Opc: TargetOpcode::G_UNMERGE_VALUES, DstOps: TmpVec, SrcOps: Op);
691}
692
693MachineInstrBuilder MachineIRBuilder::buildUnmerge(LLT Res,
694 const SrcOp &Op) {
695 unsigned NumReg = Op.getLLTTy(MRI: *getMRI()).getSizeInBits() / Res.getSizeInBits();
696 SmallVector<DstOp, 8> TmpVec(NumReg, Res);
697 return buildInstr(Opc: TargetOpcode::G_UNMERGE_VALUES, DstOps: TmpVec, SrcOps: Op);
698}
699
700MachineInstrBuilder MachineIRBuilder::buildUnmerge(ArrayRef<Register> Res,
701 const SrcOp &Op) {
702 // Unfortunately to convert from ArrayRef<Register> to ArrayRef<DstOp>,
703 // we need some temporary storage for the DstOp objects. Here we use a
704 // sufficiently large SmallVector to not go through the heap.
705 SmallVector<DstOp, 8> TmpVec(Res.begin(), Res.end());
706 assert(TmpVec.size() > 1);
707 return buildInstr(Opc: TargetOpcode::G_UNMERGE_VALUES, DstOps: TmpVec, SrcOps: Op);
708}
709
710MachineInstrBuilder MachineIRBuilder::buildBuildVector(const DstOp &Res,
711 ArrayRef<Register> Ops) {
712 // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
713 // we need some temporary storage for the DstOp objects. Here we use a
714 // sufficiently large SmallVector to not go through the heap.
715 SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end());
716 return buildInstr(Opc: TargetOpcode::G_BUILD_VECTOR, DstOps: Res, SrcOps: TmpVec);
717}
718
719MachineInstrBuilder
720MachineIRBuilder::buildBuildVectorConstant(const DstOp &Res,
721 ArrayRef<APInt> Ops) {
722 SmallVector<SrcOp> TmpVec;
723 TmpVec.reserve(N: Ops.size());
724 LLT EltTy = Res.getLLTTy(MRI: *getMRI()).getElementType();
725 for (const auto &Op : Ops)
726 TmpVec.push_back(Elt: buildConstant(Res: EltTy, Val: Op));
727 return buildInstr(Opc: TargetOpcode::G_BUILD_VECTOR, DstOps: Res, SrcOps: TmpVec);
728}
729
730MachineInstrBuilder MachineIRBuilder::buildSplatBuildVector(const DstOp &Res,
731 const SrcOp &Src) {
732 SmallVector<SrcOp, 8> TmpVec(Res.getLLTTy(MRI: *getMRI()).getNumElements(), Src);
733 return buildInstr(Opc: TargetOpcode::G_BUILD_VECTOR, DstOps: Res, SrcOps: TmpVec);
734}
735
736MachineInstrBuilder
737MachineIRBuilder::buildBuildVectorTrunc(const DstOp &Res,
738 ArrayRef<Register> Ops) {
739 // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
740 // we need some temporary storage for the DstOp objects. Here we use a
741 // sufficiently large SmallVector to not go through the heap.
742 SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end());
743 if (TmpVec[0].getLLTTy(MRI: *getMRI()).getSizeInBits() ==
744 Res.getLLTTy(MRI: *getMRI()).getElementType().getSizeInBits())
745 return buildInstr(Opc: TargetOpcode::G_BUILD_VECTOR, DstOps: Res, SrcOps: TmpVec);
746 return buildInstr(Opc: TargetOpcode::G_BUILD_VECTOR_TRUNC, DstOps: Res, SrcOps: TmpVec);
747}
748
749MachineInstrBuilder MachineIRBuilder::buildShuffleSplat(const DstOp &Res,
750 const SrcOp &Src) {
751 LLT DstTy = Res.getLLTTy(MRI: *getMRI());
752 assert(Src.getLLTTy(*getMRI()) == DstTy.getElementType() &&
753 "Expected Src to match Dst elt ty");
754 auto UndefVec = buildUndef(Res: DstTy);
755 auto Zero = buildConstant(Res: LLT::scalar(SizeInBits: 64), Val: 0);
756 auto InsElt = buildInsertVectorElement(Res: DstTy, Val: UndefVec, Elt: Src, Idx: Zero);
757 SmallVector<int, 16> ZeroMask(DstTy.getNumElements());
758 return buildShuffleVector(Res: DstTy, Src1: InsElt, Src2: UndefVec, Mask: ZeroMask);
759}
760
761MachineInstrBuilder MachineIRBuilder::buildSplatVector(const DstOp &Res,
762 const SrcOp &Src) {
763 assert(Src.getLLTTy(*getMRI()) == Res.getLLTTy(*getMRI()).getElementType() &&
764 "Expected Src to match Dst elt ty");
765 return buildInstr(Opc: TargetOpcode::G_SPLAT_VECTOR, DstOps: Res, SrcOps: Src);
766}
767
768MachineInstrBuilder MachineIRBuilder::buildShuffleVector(const DstOp &Res,
769 const SrcOp &Src1,
770 const SrcOp &Src2,
771 ArrayRef<int> Mask) {
772 LLT DstTy = Res.getLLTTy(MRI: *getMRI());
773 LLT Src1Ty = Src1.getLLTTy(MRI: *getMRI());
774 LLT Src2Ty = Src2.getLLTTy(MRI: *getMRI());
775 assert((size_t)(Src1Ty.getNumElements() + Src2Ty.getNumElements()) >=
776 Mask.size());
777 assert(DstTy.getElementType() == Src1Ty.getElementType() &&
778 DstTy.getElementType() == Src2Ty.getElementType());
779 (void)DstTy;
780 (void)Src1Ty;
781 (void)Src2Ty;
782 ArrayRef<int> MaskAlloc = getMF().allocateShuffleMask(Mask);
783 return buildInstr(Opc: TargetOpcode::G_SHUFFLE_VECTOR, DstOps: {Res}, SrcOps: {Src1, Src2})
784 .addShuffleMask(Val: MaskAlloc);
785}
786
787MachineInstrBuilder
788MachineIRBuilder::buildConcatVectors(const DstOp &Res, ArrayRef<Register> Ops) {
789 // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
790 // we need some temporary storage for the DstOp objects. Here we use a
791 // sufficiently large SmallVector to not go through the heap.
792 SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end());
793 return buildInstr(Opc: TargetOpcode::G_CONCAT_VECTORS, DstOps: Res, SrcOps: TmpVec);
794}
795
796MachineInstrBuilder MachineIRBuilder::buildInsert(const DstOp &Res,
797 const SrcOp &Src,
798 const SrcOp &Op,
799 unsigned Index) {
800 assert(Index + Op.getLLTTy(*getMRI()).getSizeInBits() <=
801 Res.getLLTTy(*getMRI()).getSizeInBits() &&
802 "insertion past the end of a register");
803
804 if (Res.getLLTTy(MRI: *getMRI()).getSizeInBits() ==
805 Op.getLLTTy(MRI: *getMRI()).getSizeInBits()) {
806 return buildCast(Dst: Res, Src: Op);
807 }
808
809 return buildInstr(Opc: TargetOpcode::G_INSERT, DstOps: Res, SrcOps: {Src, Op, uint64_t(Index)});
810}
811
812MachineInstrBuilder MachineIRBuilder::buildVScale(const DstOp &Res,
813 unsigned MinElts) {
814
815 auto IntN = IntegerType::get(C&: getMF().getFunction().getContext(),
816 NumBits: Res.getLLTTy(MRI: *getMRI()).getScalarSizeInBits());
817 ConstantInt *CI = ConstantInt::get(Ty: IntN, V: MinElts);
818 return buildVScale(Res, MinElts: *CI);
819}
820
821MachineInstrBuilder MachineIRBuilder::buildVScale(const DstOp &Res,
822 const ConstantInt &MinElts) {
823 auto VScale = buildInstr(Opcode: TargetOpcode::G_VSCALE);
824 VScale->setDebugLoc(DebugLoc());
825 Res.addDefToMIB(MRI&: *getMRI(), MIB&: VScale);
826 VScale.addCImm(Val: &MinElts);
827 return VScale;
828}
829
830MachineInstrBuilder MachineIRBuilder::buildVScale(const DstOp &Res,
831 const APInt &MinElts) {
832 ConstantInt *CI =
833 ConstantInt::get(Context&: getMF().getFunction().getContext(), V: MinElts);
834 return buildVScale(Res, MinElts: *CI);
835}
836
837static unsigned getIntrinsicOpcode(bool HasSideEffects, bool IsConvergent) {
838 if (HasSideEffects && IsConvergent)
839 return TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS;
840 if (HasSideEffects)
841 return TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS;
842 if (IsConvergent)
843 return TargetOpcode::G_INTRINSIC_CONVERGENT;
844 return TargetOpcode::G_INTRINSIC;
845}
846
847MachineInstrBuilder
848MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
849 ArrayRef<Register> ResultRegs,
850 bool HasSideEffects, bool isConvergent) {
851 auto MIB = buildInstr(Opcode: getIntrinsicOpcode(HasSideEffects, IsConvergent: isConvergent));
852 for (unsigned ResultReg : ResultRegs)
853 MIB.addDef(RegNo: ResultReg);
854 MIB.addIntrinsicID(ID);
855 return MIB;
856}
857
858MachineInstrBuilder
859MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
860 ArrayRef<Register> ResultRegs) {
861 auto Attrs = Intrinsic::getAttributes(C&: getContext(), id: ID);
862 bool HasSideEffects = !Attrs.getMemoryEffects().doesNotAccessMemory();
863 bool isConvergent = Attrs.hasFnAttr(Kind: Attribute::Convergent);
864 return buildIntrinsic(ID, ResultRegs, HasSideEffects, isConvergent);
865}
866
867MachineInstrBuilder MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
868 ArrayRef<DstOp> Results,
869 bool HasSideEffects,
870 bool isConvergent) {
871 auto MIB = buildInstr(Opcode: getIntrinsicOpcode(HasSideEffects, IsConvergent: isConvergent));
872 for (DstOp Result : Results)
873 Result.addDefToMIB(MRI&: *getMRI(), MIB);
874 MIB.addIntrinsicID(ID);
875 return MIB;
876}
877
878MachineInstrBuilder MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
879 ArrayRef<DstOp> Results) {
880 auto Attrs = Intrinsic::getAttributes(C&: getContext(), id: ID);
881 bool HasSideEffects = !Attrs.getMemoryEffects().doesNotAccessMemory();
882 bool isConvergent = Attrs.hasFnAttr(Kind: Attribute::Convergent);
883 return buildIntrinsic(ID, Results, HasSideEffects, isConvergent);
884}
885
886MachineInstrBuilder
887MachineIRBuilder::buildTrunc(const DstOp &Res, const SrcOp &Op,
888 std::optional<unsigned> Flags) {
889 return buildInstr(Opc: TargetOpcode::G_TRUNC, DstOps: Res, SrcOps: Op, Flags);
890}
891
892MachineInstrBuilder
893MachineIRBuilder::buildFPTrunc(const DstOp &Res, const SrcOp &Op,
894 std::optional<unsigned> Flags) {
895 return buildInstr(Opc: TargetOpcode::G_FPTRUNC, DstOps: Res, SrcOps: Op, Flags);
896}
897
898MachineInstrBuilder MachineIRBuilder::buildICmp(CmpInst::Predicate Pred,
899 const DstOp &Res,
900 const SrcOp &Op0,
901 const SrcOp &Op1) {
902 return buildInstr(Opc: TargetOpcode::G_ICMP, DstOps: Res, SrcOps: {Pred, Op0, Op1});
903}
904
905MachineInstrBuilder MachineIRBuilder::buildFCmp(CmpInst::Predicate Pred,
906 const DstOp &Res,
907 const SrcOp &Op0,
908 const SrcOp &Op1,
909 std::optional<unsigned> Flags) {
910
911 return buildInstr(Opc: TargetOpcode::G_FCMP, DstOps: Res, SrcOps: {Pred, Op0, Op1}, Flags);
912}
913
914MachineInstrBuilder MachineIRBuilder::buildSCmp(const DstOp &Res,
915 const SrcOp &Op0,
916 const SrcOp &Op1) {
917 return buildInstr(Opc: TargetOpcode::G_SCMP, DstOps: Res, SrcOps: {Op0, Op1});
918}
919
920MachineInstrBuilder MachineIRBuilder::buildUCmp(const DstOp &Res,
921 const SrcOp &Op0,
922 const SrcOp &Op1) {
923 return buildInstr(Opc: TargetOpcode::G_UCMP, DstOps: Res, SrcOps: {Op0, Op1});
924}
925
926MachineInstrBuilder
927MachineIRBuilder::buildSelect(const DstOp &Res, const SrcOp &Tst,
928 const SrcOp &Op0, const SrcOp &Op1,
929 std::optional<unsigned> Flags) {
930
931 return buildInstr(Opc: TargetOpcode::G_SELECT, DstOps: {Res}, SrcOps: {Tst, Op0, Op1}, Flags);
932}
933
934MachineInstrBuilder MachineIRBuilder::buildInsertSubvector(const DstOp &Res,
935 const SrcOp &Src0,
936 const SrcOp &Src1,
937 unsigned Idx) {
938 return buildInstr(Opc: TargetOpcode::G_INSERT_SUBVECTOR, DstOps: Res,
939 SrcOps: {Src0, Src1, uint64_t(Idx)});
940}
941
942MachineInstrBuilder MachineIRBuilder::buildExtractSubvector(const DstOp &Res,
943 const SrcOp &Src,
944 unsigned Idx) {
945 return buildInstr(Opc: TargetOpcode::G_INSERT_SUBVECTOR, DstOps: Res,
946 SrcOps: {Src, uint64_t(Idx)});
947}
948
949MachineInstrBuilder
950MachineIRBuilder::buildInsertVectorElement(const DstOp &Res, const SrcOp &Val,
951 const SrcOp &Elt, const SrcOp &Idx) {
952 return buildInstr(Opc: TargetOpcode::G_INSERT_VECTOR_ELT, DstOps: Res, SrcOps: {Val, Elt, Idx});
953}
954
955MachineInstrBuilder
956MachineIRBuilder::buildExtractVectorElement(const DstOp &Res, const SrcOp &Val,
957 const SrcOp &Idx) {
958 return buildInstr(Opc: TargetOpcode::G_EXTRACT_VECTOR_ELT, DstOps: Res, SrcOps: {Val, Idx});
959}
960
961MachineInstrBuilder MachineIRBuilder::buildAtomicCmpXchgWithSuccess(
962 const DstOp &OldValRes, const DstOp &SuccessRes, const SrcOp &Addr,
963 const SrcOp &CmpVal, const SrcOp &NewVal, MachineMemOperand &MMO) {
964#ifndef NDEBUG
965 LLT OldValResTy = OldValRes.getLLTTy(*getMRI());
966 LLT SuccessResTy = SuccessRes.getLLTTy(*getMRI());
967 LLT AddrTy = Addr.getLLTTy(*getMRI());
968 LLT CmpValTy = CmpVal.getLLTTy(*getMRI());
969 LLT NewValTy = NewVal.getLLTTy(*getMRI());
970 assert(OldValResTy.isScalar() && "invalid operand type");
971 assert(SuccessResTy.isScalar() && "invalid operand type");
972 assert(AddrTy.isPointer() && "invalid operand type");
973 assert(CmpValTy.isValid() && "invalid operand type");
974 assert(NewValTy.isValid() && "invalid operand type");
975 assert(OldValResTy == CmpValTy && "type mismatch");
976 assert(OldValResTy == NewValTy && "type mismatch");
977#endif
978
979 auto MIB = buildInstr(Opcode: TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS);
980 OldValRes.addDefToMIB(MRI&: *getMRI(), MIB);
981 SuccessRes.addDefToMIB(MRI&: *getMRI(), MIB);
982 Addr.addSrcToMIB(MIB);
983 CmpVal.addSrcToMIB(MIB);
984 NewVal.addSrcToMIB(MIB);
985 MIB.addMemOperand(MMO: &MMO);
986 return MIB;
987}
988
989MachineInstrBuilder
990MachineIRBuilder::buildAtomicCmpXchg(const DstOp &OldValRes, const SrcOp &Addr,
991 const SrcOp &CmpVal, const SrcOp &NewVal,
992 MachineMemOperand &MMO) {
993#ifndef NDEBUG
994 LLT OldValResTy = OldValRes.getLLTTy(*getMRI());
995 LLT AddrTy = Addr.getLLTTy(*getMRI());
996 LLT CmpValTy = CmpVal.getLLTTy(*getMRI());
997 LLT NewValTy = NewVal.getLLTTy(*getMRI());
998 assert(OldValResTy.isScalar() && "invalid operand type");
999 assert(AddrTy.isPointer() && "invalid operand type");
1000 assert(CmpValTy.isValid() && "invalid operand type");
1001 assert(NewValTy.isValid() && "invalid operand type");
1002 assert(OldValResTy == CmpValTy && "type mismatch");
1003 assert(OldValResTy == NewValTy && "type mismatch");
1004#endif
1005
1006 auto MIB = buildInstr(Opcode: TargetOpcode::G_ATOMIC_CMPXCHG);
1007 OldValRes.addDefToMIB(MRI&: *getMRI(), MIB);
1008 Addr.addSrcToMIB(MIB);
1009 CmpVal.addSrcToMIB(MIB);
1010 NewVal.addSrcToMIB(MIB);
1011 MIB.addMemOperand(MMO: &MMO);
1012 return MIB;
1013}
1014
1015MachineInstrBuilder MachineIRBuilder::buildAtomicRMW(
1016 unsigned Opcode, const DstOp &OldValRes,
1017 const SrcOp &Addr, const SrcOp &Val,
1018 MachineMemOperand &MMO) {
1019
1020#ifndef NDEBUG
1021 LLT OldValResTy = OldValRes.getLLTTy(*getMRI());
1022 LLT AddrTy = Addr.getLLTTy(*getMRI());
1023 LLT ValTy = Val.getLLTTy(*getMRI());
1024 assert(AddrTy.isPointer() && "invalid operand type");
1025 assert(ValTy.isValid() && "invalid operand type");
1026 assert(OldValResTy == ValTy && "type mismatch");
1027 assert(MMO.isAtomic() && "not atomic mem operand");
1028#endif
1029
1030 auto MIB = buildInstr(Opcode);
1031 OldValRes.addDefToMIB(MRI&: *getMRI(), MIB);
1032 Addr.addSrcToMIB(MIB);
1033 Val.addSrcToMIB(MIB);
1034 MIB.addMemOperand(MMO: &MMO);
1035 return MIB;
1036}
1037
1038MachineInstrBuilder
1039MachineIRBuilder::buildAtomicRMWXchg(Register OldValRes, Register Addr,
1040 Register Val, MachineMemOperand &MMO) {
1041 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_XCHG, OldValRes, Addr, Val,
1042 MMO);
1043}
1044MachineInstrBuilder
1045MachineIRBuilder::buildAtomicRMWAdd(Register OldValRes, Register Addr,
1046 Register Val, MachineMemOperand &MMO) {
1047 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_ADD, OldValRes, Addr, Val,
1048 MMO);
1049}
1050MachineInstrBuilder
1051MachineIRBuilder::buildAtomicRMWSub(Register OldValRes, Register Addr,
1052 Register Val, MachineMemOperand &MMO) {
1053 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_SUB, OldValRes, Addr, Val,
1054 MMO);
1055}
1056MachineInstrBuilder
1057MachineIRBuilder::buildAtomicRMWAnd(Register OldValRes, Register Addr,
1058 Register Val, MachineMemOperand &MMO) {
1059 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_AND, OldValRes, Addr, Val,
1060 MMO);
1061}
1062MachineInstrBuilder
1063MachineIRBuilder::buildAtomicRMWNand(Register OldValRes, Register Addr,
1064 Register Val, MachineMemOperand &MMO) {
1065 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_NAND, OldValRes, Addr, Val,
1066 MMO);
1067}
1068MachineInstrBuilder MachineIRBuilder::buildAtomicRMWOr(Register OldValRes,
1069 Register Addr,
1070 Register Val,
1071 MachineMemOperand &MMO) {
1072 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_OR, OldValRes, Addr, Val,
1073 MMO);
1074}
1075MachineInstrBuilder
1076MachineIRBuilder::buildAtomicRMWXor(Register OldValRes, Register Addr,
1077 Register Val, MachineMemOperand &MMO) {
1078 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_XOR, OldValRes, Addr, Val,
1079 MMO);
1080}
1081MachineInstrBuilder
1082MachineIRBuilder::buildAtomicRMWMax(Register OldValRes, Register Addr,
1083 Register Val, MachineMemOperand &MMO) {
1084 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_MAX, OldValRes, Addr, Val,
1085 MMO);
1086}
1087MachineInstrBuilder
1088MachineIRBuilder::buildAtomicRMWMin(Register OldValRes, Register Addr,
1089 Register Val, MachineMemOperand &MMO) {
1090 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_MIN, OldValRes, Addr, Val,
1091 MMO);
1092}
1093MachineInstrBuilder
1094MachineIRBuilder::buildAtomicRMWUmax(Register OldValRes, Register Addr,
1095 Register Val, MachineMemOperand &MMO) {
1096 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_UMAX, OldValRes, Addr, Val,
1097 MMO);
1098}
1099MachineInstrBuilder
1100MachineIRBuilder::buildAtomicRMWUmin(Register OldValRes, Register Addr,
1101 Register Val, MachineMemOperand &MMO) {
1102 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_UMIN, OldValRes, Addr, Val,
1103 MMO);
1104}
1105
1106MachineInstrBuilder
1107MachineIRBuilder::buildAtomicRMWFAdd(
1108 const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val,
1109 MachineMemOperand &MMO) {
1110 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_FADD, OldValRes, Addr, Val,
1111 MMO);
1112}
1113
1114MachineInstrBuilder
1115MachineIRBuilder::buildAtomicRMWFSub(const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val,
1116 MachineMemOperand &MMO) {
1117 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_FSUB, OldValRes, Addr, Val,
1118 MMO);
1119}
1120
1121MachineInstrBuilder
1122MachineIRBuilder::buildAtomicRMWFMax(const DstOp &OldValRes, const SrcOp &Addr,
1123 const SrcOp &Val, MachineMemOperand &MMO) {
1124 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_FMAX, OldValRes, Addr, Val,
1125 MMO);
1126}
1127
1128MachineInstrBuilder
1129MachineIRBuilder::buildAtomicRMWFMin(const DstOp &OldValRes, const SrcOp &Addr,
1130 const SrcOp &Val, MachineMemOperand &MMO) {
1131 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_FMIN, OldValRes, Addr, Val,
1132 MMO);
1133}
1134
1135MachineInstrBuilder
1136MachineIRBuilder::buildFence(unsigned Ordering, unsigned Scope) {
1137 return buildInstr(Opcode: TargetOpcode::G_FENCE)
1138 .addImm(Val: Ordering)
1139 .addImm(Val: Scope);
1140}
1141
1142MachineInstrBuilder MachineIRBuilder::buildPrefetch(const SrcOp &Addr,
1143 unsigned RW,
1144 unsigned Locality,
1145 unsigned CacheType,
1146 MachineMemOperand &MMO) {
1147 auto MIB = buildInstr(Opcode: TargetOpcode::G_PREFETCH);
1148 Addr.addSrcToMIB(MIB);
1149 MIB.addImm(Val: RW).addImm(Val: Locality).addImm(Val: CacheType);
1150 MIB.addMemOperand(MMO: &MMO);
1151 return MIB;
1152}
1153
1154MachineInstrBuilder
1155MachineIRBuilder::buildBlockAddress(Register Res, const BlockAddress *BA) {
1156#ifndef NDEBUG
1157 assert(getMRI()->getType(Res).isPointer() && "invalid res type");
1158#endif
1159
1160 return buildInstr(Opcode: TargetOpcode::G_BLOCK_ADDR).addDef(RegNo: Res).addBlockAddress(BA);
1161}
1162
1163void MachineIRBuilder::validateTruncExt(const LLT DstTy, const LLT SrcTy,
1164 bool IsExtend) {
1165#ifndef NDEBUG
1166 if (DstTy.isVector()) {
1167 assert(SrcTy.isVector() && "mismatched cast between vector and non-vector");
1168 assert(SrcTy.getElementCount() == DstTy.getElementCount() &&
1169 "different number of elements in a trunc/ext");
1170 } else
1171 assert(DstTy.isScalar() && SrcTy.isScalar() && "invalid extend/trunc");
1172
1173 if (IsExtend)
1174 assert(TypeSize::isKnownGT(DstTy.getSizeInBits(), SrcTy.getSizeInBits()) &&
1175 "invalid narrowing extend");
1176 else
1177 assert(TypeSize::isKnownLT(DstTy.getSizeInBits(), SrcTy.getSizeInBits()) &&
1178 "invalid widening trunc");
1179#endif
1180}
1181
1182void MachineIRBuilder::validateSelectOp(const LLT ResTy, const LLT TstTy,
1183 const LLT Op0Ty, const LLT Op1Ty) {
1184#ifndef NDEBUG
1185 assert((ResTy.isScalar() || ResTy.isVector() || ResTy.isPointer()) &&
1186 "invalid operand type");
1187 assert((ResTy == Op0Ty && ResTy == Op1Ty) && "type mismatch");
1188 if (ResTy.isScalar() || ResTy.isPointer())
1189 assert(TstTy.isScalar() && "type mismatch");
1190 else
1191 assert((TstTy.isScalar() ||
1192 (TstTy.isVector() &&
1193 TstTy.getElementCount() == Op0Ty.getElementCount())) &&
1194 "type mismatch");
1195#endif
1196}
1197
1198MachineInstrBuilder
1199MachineIRBuilder::buildInstr(unsigned Opc, ArrayRef<DstOp> DstOps,
1200 ArrayRef<SrcOp> SrcOps,
1201 std::optional<unsigned> Flags) {
1202 switch (Opc) {
1203 default:
1204 break;
1205 case TargetOpcode::G_SELECT: {
1206 assert(DstOps.size() == 1 && "Invalid select");
1207 assert(SrcOps.size() == 3 && "Invalid select");
1208 validateSelectOp(
1209 ResTy: DstOps[0].getLLTTy(MRI: *getMRI()), TstTy: SrcOps[0].getLLTTy(MRI: *getMRI()),
1210 Op0Ty: SrcOps[1].getLLTTy(MRI: *getMRI()), Op1Ty: SrcOps[2].getLLTTy(MRI: *getMRI()));
1211 break;
1212 }
1213 case TargetOpcode::G_FNEG:
1214 case TargetOpcode::G_ABS:
1215 // All these are unary ops.
1216 assert(DstOps.size() == 1 && "Invalid Dst");
1217 assert(SrcOps.size() == 1 && "Invalid Srcs");
1218 validateUnaryOp(Res: DstOps[0].getLLTTy(MRI: *getMRI()),
1219 Op0: SrcOps[0].getLLTTy(MRI: *getMRI()));
1220 break;
1221 case TargetOpcode::G_ADD:
1222 case TargetOpcode::G_AND:
1223 case TargetOpcode::G_MUL:
1224 case TargetOpcode::G_OR:
1225 case TargetOpcode::G_SUB:
1226 case TargetOpcode::G_XOR:
1227 case TargetOpcode::G_UDIV:
1228 case TargetOpcode::G_SDIV:
1229 case TargetOpcode::G_UREM:
1230 case TargetOpcode::G_SREM:
1231 case TargetOpcode::G_SMIN:
1232 case TargetOpcode::G_SMAX:
1233 case TargetOpcode::G_UMIN:
1234 case TargetOpcode::G_UMAX:
1235 case TargetOpcode::G_UADDSAT:
1236 case TargetOpcode::G_SADDSAT:
1237 case TargetOpcode::G_USUBSAT:
1238 case TargetOpcode::G_SSUBSAT: {
1239 // All these are binary ops.
1240 assert(DstOps.size() == 1 && "Invalid Dst");
1241 assert(SrcOps.size() == 2 && "Invalid Srcs");
1242 validateBinaryOp(Res: DstOps[0].getLLTTy(MRI: *getMRI()),
1243 Op0: SrcOps[0].getLLTTy(MRI: *getMRI()),
1244 Op1: SrcOps[1].getLLTTy(MRI: *getMRI()));
1245 break;
1246 }
1247 case TargetOpcode::G_SHL:
1248 case TargetOpcode::G_ASHR:
1249 case TargetOpcode::G_LSHR:
1250 case TargetOpcode::G_USHLSAT:
1251 case TargetOpcode::G_SSHLSAT: {
1252 assert(DstOps.size() == 1 && "Invalid Dst");
1253 assert(SrcOps.size() == 2 && "Invalid Srcs");
1254 validateShiftOp(Res: DstOps[0].getLLTTy(MRI: *getMRI()),
1255 Op0: SrcOps[0].getLLTTy(MRI: *getMRI()),
1256 Op1: SrcOps[1].getLLTTy(MRI: *getMRI()));
1257 break;
1258 }
1259 case TargetOpcode::G_SEXT:
1260 case TargetOpcode::G_ZEXT:
1261 case TargetOpcode::G_ANYEXT:
1262 assert(DstOps.size() == 1 && "Invalid Dst");
1263 assert(SrcOps.size() == 1 && "Invalid Srcs");
1264 validateTruncExt(DstTy: DstOps[0].getLLTTy(MRI: *getMRI()),
1265 SrcTy: SrcOps[0].getLLTTy(MRI: *getMRI()), IsExtend: true);
1266 break;
1267 case TargetOpcode::G_TRUNC:
1268 case TargetOpcode::G_FPTRUNC: {
1269 assert(DstOps.size() == 1 && "Invalid Dst");
1270 assert(SrcOps.size() == 1 && "Invalid Srcs");
1271 validateTruncExt(DstTy: DstOps[0].getLLTTy(MRI: *getMRI()),
1272 SrcTy: SrcOps[0].getLLTTy(MRI: *getMRI()), IsExtend: false);
1273 break;
1274 }
1275 case TargetOpcode::G_BITCAST: {
1276 assert(DstOps.size() == 1 && "Invalid Dst");
1277 assert(SrcOps.size() == 1 && "Invalid Srcs");
1278 assert(DstOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1279 SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() && "invalid bitcast");
1280 break;
1281 }
1282 case TargetOpcode::COPY:
1283 assert(DstOps.size() == 1 && "Invalid Dst");
1284 // If the caller wants to add a subreg source it has to be done separately
1285 // so we may not have any SrcOps at this point yet.
1286 break;
1287 case TargetOpcode::G_FCMP:
1288 case TargetOpcode::G_ICMP: {
1289 assert(DstOps.size() == 1 && "Invalid Dst Operands");
1290 assert(SrcOps.size() == 3 && "Invalid Src Operands");
1291 // For F/ICMP, the first src operand is the predicate, followed by
1292 // the two comparands.
1293 assert(SrcOps[0].getSrcOpKind() == SrcOp::SrcType::Ty_Predicate &&
1294 "Expecting predicate");
1295 assert([&]() -> bool {
1296 CmpInst::Predicate Pred = SrcOps[0].getPredicate();
1297 return Opc == TargetOpcode::G_ICMP ? CmpInst::isIntPredicate(Pred)
1298 : CmpInst::isFPPredicate(Pred);
1299 }() && "Invalid predicate");
1300 assert(SrcOps[1].getLLTTy(*getMRI()) == SrcOps[2].getLLTTy(*getMRI()) &&
1301 "Type mismatch");
1302 assert([&]() -> bool {
1303 LLT Op0Ty = SrcOps[1].getLLTTy(*getMRI());
1304 LLT DstTy = DstOps[0].getLLTTy(*getMRI());
1305 if (Op0Ty.isScalar() || Op0Ty.isPointer())
1306 return DstTy.isScalar();
1307 else
1308 return DstTy.isVector() &&
1309 DstTy.getElementCount() == Op0Ty.getElementCount();
1310 }() && "Type Mismatch");
1311 break;
1312 }
1313 case TargetOpcode::G_UNMERGE_VALUES: {
1314 assert(!DstOps.empty() && "Invalid trivial sequence");
1315 assert(SrcOps.size() == 1 && "Invalid src for Unmerge");
1316 assert(llvm::all_of(DstOps,
1317 [&, this](const DstOp &Op) {
1318 return Op.getLLTTy(*getMRI()) ==
1319 DstOps[0].getLLTTy(*getMRI());
1320 }) &&
1321 "type mismatch in output list");
1322 assert((TypeSize::ScalarTy)DstOps.size() *
1323 DstOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1324 SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1325 "input operands do not cover output register");
1326 break;
1327 }
1328 case TargetOpcode::G_MERGE_VALUES: {
1329 assert(SrcOps.size() >= 2 && "invalid trivial sequence");
1330 assert(DstOps.size() == 1 && "Invalid Dst");
1331 assert(llvm::all_of(SrcOps,
1332 [&, this](const SrcOp &Op) {
1333 return Op.getLLTTy(*getMRI()) ==
1334 SrcOps[0].getLLTTy(*getMRI());
1335 }) &&
1336 "type mismatch in input list");
1337 assert((TypeSize::ScalarTy)SrcOps.size() *
1338 SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1339 DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1340 "input operands do not cover output register");
1341 assert(!DstOps[0].getLLTTy(*getMRI()).isVector() &&
1342 "vectors should be built with G_CONCAT_VECTOR or G_BUILD_VECTOR");
1343 break;
1344 }
1345 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
1346 assert(DstOps.size() == 1 && "Invalid Dst size");
1347 assert(SrcOps.size() == 2 && "Invalid Src size");
1348 assert(SrcOps[0].getLLTTy(*getMRI()).isVector() && "Invalid operand type");
1349 assert((DstOps[0].getLLTTy(*getMRI()).isScalar() ||
1350 DstOps[0].getLLTTy(*getMRI()).isPointer()) &&
1351 "Invalid operand type");
1352 assert(SrcOps[1].getLLTTy(*getMRI()).isScalar() && "Invalid operand type");
1353 assert(SrcOps[0].getLLTTy(*getMRI()).getElementType() ==
1354 DstOps[0].getLLTTy(*getMRI()) &&
1355 "Type mismatch");
1356 break;
1357 }
1358 case TargetOpcode::G_INSERT_VECTOR_ELT: {
1359 assert(DstOps.size() == 1 && "Invalid dst size");
1360 assert(SrcOps.size() == 3 && "Invalid src size");
1361 assert(DstOps[0].getLLTTy(*getMRI()).isVector() &&
1362 SrcOps[0].getLLTTy(*getMRI()).isVector() && "Invalid operand type");
1363 assert(DstOps[0].getLLTTy(*getMRI()).getElementType() ==
1364 SrcOps[1].getLLTTy(*getMRI()) &&
1365 "Type mismatch");
1366 assert(SrcOps[2].getLLTTy(*getMRI()).isScalar() && "Invalid index");
1367 assert(DstOps[0].getLLTTy(*getMRI()).getElementCount() ==
1368 SrcOps[0].getLLTTy(*getMRI()).getElementCount() &&
1369 "Type mismatch");
1370 break;
1371 }
1372 case TargetOpcode::G_BUILD_VECTOR: {
1373 assert((!SrcOps.empty() || SrcOps.size() < 2) &&
1374 "Must have at least 2 operands");
1375 assert(DstOps.size() == 1 && "Invalid DstOps");
1376 assert(DstOps[0].getLLTTy(*getMRI()).isVector() &&
1377 "Res type must be a vector");
1378 assert(llvm::all_of(SrcOps,
1379 [&, this](const SrcOp &Op) {
1380 return Op.getLLTTy(*getMRI()) ==
1381 SrcOps[0].getLLTTy(*getMRI());
1382 }) &&
1383 "type mismatch in input list");
1384 assert((TypeSize::ScalarTy)SrcOps.size() *
1385 SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1386 DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1387 "input scalars do not exactly cover the output vector register");
1388 break;
1389 }
1390 case TargetOpcode::G_BUILD_VECTOR_TRUNC: {
1391 assert((!SrcOps.empty() || SrcOps.size() < 2) &&
1392 "Must have at least 2 operands");
1393 assert(DstOps.size() == 1 && "Invalid DstOps");
1394 assert(DstOps[0].getLLTTy(*getMRI()).isVector() &&
1395 "Res type must be a vector");
1396 assert(llvm::all_of(SrcOps,
1397 [&, this](const SrcOp &Op) {
1398 return Op.getLLTTy(*getMRI()) ==
1399 SrcOps[0].getLLTTy(*getMRI());
1400 }) &&
1401 "type mismatch in input list");
1402 break;
1403 }
1404 case TargetOpcode::G_CONCAT_VECTORS: {
1405 assert(DstOps.size() == 1 && "Invalid DstOps");
1406 assert((!SrcOps.empty() || SrcOps.size() < 2) &&
1407 "Must have at least 2 operands");
1408 assert(llvm::all_of(SrcOps,
1409 [&, this](const SrcOp &Op) {
1410 return (Op.getLLTTy(*getMRI()).isVector() &&
1411 Op.getLLTTy(*getMRI()) ==
1412 SrcOps[0].getLLTTy(*getMRI()));
1413 }) &&
1414 "type mismatch in input list");
1415 assert((TypeSize::ScalarTy)SrcOps.size() *
1416 SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1417 DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1418 "input vectors do not exactly cover the output vector register");
1419 break;
1420 }
1421 case TargetOpcode::G_UADDE: {
1422 assert(DstOps.size() == 2 && "Invalid no of dst operands");
1423 assert(SrcOps.size() == 3 && "Invalid no of src operands");
1424 assert(DstOps[0].getLLTTy(*getMRI()).isScalar() && "Invalid operand");
1425 assert((DstOps[0].getLLTTy(*getMRI()) == SrcOps[0].getLLTTy(*getMRI())) &&
1426 (DstOps[0].getLLTTy(*getMRI()) == SrcOps[1].getLLTTy(*getMRI())) &&
1427 "Invalid operand");
1428 assert(DstOps[1].getLLTTy(*getMRI()).isScalar() && "Invalid operand");
1429 assert(DstOps[1].getLLTTy(*getMRI()) == SrcOps[2].getLLTTy(*getMRI()) &&
1430 "type mismatch");
1431 break;
1432 }
1433 }
1434
1435 auto MIB = buildInstr(Opcode: Opc);
1436 for (const DstOp &Op : DstOps)
1437 Op.addDefToMIB(MRI&: *getMRI(), MIB);
1438 for (const SrcOp &Op : SrcOps)
1439 Op.addSrcToMIB(MIB);
1440 if (Flags)
1441 MIB->setFlags(*Flags);
1442 return MIB;
1443}
1444