1//===-- llvm/CodeGen/GlobalISel/MachineIRBuilder.cpp - MIBuilder--*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the MachineIRBuidler class.
10//===----------------------------------------------------------------------===//
11#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
12#include "llvm/CodeGen/MachineFunction.h"
13#include "llvm/CodeGen/MachineInstr.h"
14#include "llvm/CodeGen/MachineInstrBuilder.h"
15#include "llvm/CodeGen/MachineRegisterInfo.h"
16#include "llvm/CodeGen/TargetInstrInfo.h"
17#include "llvm/CodeGen/TargetLowering.h"
18#include "llvm/CodeGen/TargetOpcodes.h"
19#include "llvm/CodeGen/TargetSubtargetInfo.h"
20#include "llvm/IR/DebugInfoMetadata.h"
21
22using namespace llvm;
23
24void MachineIRBuilder::setMF(MachineFunction &MF) {
25 State.MF = &MF;
26 State.MBB = nullptr;
27 State.MRI = &MF.getRegInfo();
28 State.TII = MF.getSubtarget().getInstrInfo();
29 State.DL = DebugLoc();
30 State.PCSections = nullptr;
31 State.MMRA = nullptr;
32 State.II = MachineBasicBlock::iterator();
33 State.Observer = nullptr;
34}
35
36//------------------------------------------------------------------------------
37// Build instruction variants.
38//------------------------------------------------------------------------------
39
40MachineInstrBuilder MachineIRBuilder::buildInstrNoInsert(unsigned Opcode) {
41 return BuildMI(
42 MF&: getMF(),
43 MIMD: {getDL(), getPCSections(), getMMRAMetadata(), getDeactivationSymbol()},
44 MCID: getTII().get(Opcode));
45}
46
47MachineInstrBuilder MachineIRBuilder::insertInstr(MachineInstrBuilder MIB) {
48 getMBB().insert(I: getInsertPt(), MI: MIB);
49 recordInsertion(InsertedInstr: MIB);
50 return MIB;
51}
52
53MachineInstrBuilder
54MachineIRBuilder::buildDirectDbgValue(Register Reg, const MDNode *Variable,
55 const MDNode *Expr) {
56 assert(isa<DILocalVariable>(Variable) && "not a variable");
57 assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
58 assert(
59 cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
60 "Expected inlined-at fields to agree");
61 return insertInstr(MIB: BuildMI(MF&: getMF(), DL: getDL(),
62 MCID: getTII().get(Opcode: TargetOpcode::DBG_VALUE),
63 /*IsIndirect*/ false, Reg, Variable, Expr));
64}
65
66MachineInstrBuilder
67MachineIRBuilder::buildIndirectDbgValue(Register Reg, const MDNode *Variable,
68 const MDNode *Expr) {
69 assert(isa<DILocalVariable>(Variable) && "not a variable");
70 assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
71 assert(
72 cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
73 "Expected inlined-at fields to agree");
74 return insertInstr(MIB: BuildMI(MF&: getMF(), DL: getDL(),
75 MCID: getTII().get(Opcode: TargetOpcode::DBG_VALUE),
76 /*IsIndirect*/ true, Reg, Variable, Expr));
77}
78
79MachineInstrBuilder MachineIRBuilder::buildFIDbgValue(int FI,
80 const MDNode *Variable,
81 const MDNode *Expr) {
82 assert(isa<DILocalVariable>(Variable) && "not a variable");
83 assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
84 assert(
85 cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
86 "Expected inlined-at fields to agree");
87 return insertInstr(MIB: buildInstrNoInsert(Opcode: TargetOpcode::DBG_VALUE)
88 .addFrameIndex(Idx: FI)
89 .addImm(Val: 0)
90 .addMetadata(MD: Variable)
91 .addMetadata(MD: Expr));
92}
93
94MachineInstrBuilder MachineIRBuilder::buildConstDbgValue(const Constant &C,
95 const MDNode *Variable,
96 const MDNode *Expr) {
97 assert(isa<DILocalVariable>(Variable) && "not a variable");
98 assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
99 assert(
100 cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
101 "Expected inlined-at fields to agree");
102 auto MIB = buildInstrNoInsert(Opcode: TargetOpcode::DBG_VALUE);
103
104 auto *NumericConstant = [&] () -> const Constant* {
105 if (const auto *CE = dyn_cast<ConstantExpr>(Val: &C))
106 if (CE->getOpcode() == Instruction::IntToPtr)
107 return CE->getOperand(i_nocapture: 0);
108 return &C;
109 }();
110
111 if (auto *CI = dyn_cast<ConstantInt>(Val: NumericConstant)) {
112 if (CI->getBitWidth() > 64)
113 MIB.addCImm(Val: CI);
114 else if (CI->getBitWidth() == 1)
115 MIB.addImm(Val: CI->getZExtValue());
116 else
117 MIB.addImm(Val: CI->getSExtValue());
118 } else if (auto *CFP = dyn_cast<ConstantFP>(Val: NumericConstant)) {
119 MIB.addFPImm(Val: CFP);
120 } else if (isa<ConstantPointerNull>(Val: NumericConstant)) {
121 MIB.addImm(Val: 0);
122 } else {
123 // Insert $noreg if we didn't find a usable constant and had to drop it.
124 MIB.addReg(RegNo: Register());
125 }
126
127 MIB.addImm(Val: 0).addMetadata(MD: Variable).addMetadata(MD: Expr);
128 return insertInstr(MIB);
129}
130
131MachineInstrBuilder MachineIRBuilder::buildDbgLabel(const MDNode *Label) {
132 assert(isa<DILabel>(Label) && "not a label");
133 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(State.DL) &&
134 "Expected inlined-at fields to agree");
135 auto MIB = buildInstr(Opcode: TargetOpcode::DBG_LABEL);
136
137 return MIB.addMetadata(MD: Label);
138}
139
140MachineInstrBuilder MachineIRBuilder::buildDynStackAlloc(const DstOp &Res,
141 const SrcOp &Size,
142 Align Alignment) {
143 assert(Res.getLLTTy(*getMRI()).isPointer() && "expected ptr dst type");
144 auto MIB = buildInstr(Opcode: TargetOpcode::G_DYN_STACKALLOC);
145 Res.addDefToMIB(MRI&: *getMRI(), MIB);
146 Size.addSrcToMIB(MIB);
147 MIB.addImm(Val: Alignment.value());
148 return MIB;
149}
150
151MachineInstrBuilder MachineIRBuilder::buildFrameIndex(const DstOp &Res,
152 int Idx) {
153 assert(Res.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
154 auto MIB = buildInstr(Opcode: TargetOpcode::G_FRAME_INDEX);
155 Res.addDefToMIB(MRI&: *getMRI(), MIB);
156 MIB.addFrameIndex(Idx);
157 return MIB;
158}
159
160MachineInstrBuilder MachineIRBuilder::buildGlobalValue(const DstOp &Res,
161 const GlobalValue *GV) {
162 assert(Res.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
163 assert(Res.getLLTTy(*getMRI()).getAddressSpace() ==
164 GV->getType()->getAddressSpace() &&
165 "address space mismatch");
166
167 auto MIB = buildInstr(Opcode: TargetOpcode::G_GLOBAL_VALUE);
168 Res.addDefToMIB(MRI&: *getMRI(), MIB);
169 MIB.addGlobalAddress(GV);
170 return MIB;
171}
172
173MachineInstrBuilder MachineIRBuilder::buildConstantPool(const DstOp &Res,
174 unsigned Idx) {
175 assert(Res.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
176 auto MIB = buildInstr(Opcode: TargetOpcode::G_CONSTANT_POOL);
177 Res.addDefToMIB(MRI&: *getMRI(), MIB);
178 MIB.addConstantPoolIndex(Idx);
179 return MIB;
180}
181
182MachineInstrBuilder MachineIRBuilder::buildJumpTable(const LLT PtrTy,
183 unsigned JTI) {
184 return buildInstr(Opc: TargetOpcode::G_JUMP_TABLE, DstOps: {PtrTy}, SrcOps: {})
185 .addJumpTableIndex(Idx: JTI);
186}
187
188void MachineIRBuilder::validateUnaryOp(const LLT Res, const LLT Op0) {
189 assert((Res.isScalar() || Res.isVector()) && "invalid operand type");
190 assert((Res == Op0) && "type mismatch");
191}
192
193void MachineIRBuilder::validateBinaryOp(const LLT Res, const LLT Op0,
194 const LLT Op1) {
195 assert((Res.isScalar() || Res.isVector()) && "invalid operand type");
196 assert((Res == Op0 && Res == Op1) && "type mismatch");
197}
198
199void MachineIRBuilder::validateShiftOp(const LLT Res, const LLT Op0,
200 const LLT Op1) {
201 assert((Res.isScalar() || Res.isVector()) && "invalid operand type");
202 assert((Res == Op0) && "type mismatch");
203}
204
205MachineInstrBuilder
206MachineIRBuilder::buildPtrAdd(const DstOp &Res, const SrcOp &Op0,
207 const SrcOp &Op1, std::optional<unsigned> Flags) {
208 assert(Res.getLLTTy(*getMRI()).isPointerOrPointerVector() &&
209 Res.getLLTTy(*getMRI()) == Op0.getLLTTy(*getMRI()) && "type mismatch");
210 assert(Op1.getLLTTy(*getMRI()).getScalarType().isScalar() && "invalid offset type");
211
212 return buildInstr(Opc: TargetOpcode::G_PTR_ADD, DstOps: {Res}, SrcOps: {Op0, Op1}, Flags);
213}
214
215MachineInstrBuilder MachineIRBuilder::buildObjectPtrOffset(const DstOp &Res,
216 const SrcOp &Op0,
217 const SrcOp &Op1) {
218 return buildPtrAdd(Res, Op0, Op1,
219 Flags: MachineInstr::MIFlag::NoUWrap |
220 MachineInstr::MIFlag::InBounds);
221}
222
223std::optional<MachineInstrBuilder>
224MachineIRBuilder::materializePtrAdd(Register &Res, Register Op0,
225 const LLT ValueTy, uint64_t Value,
226 std::optional<unsigned> Flags) {
227 assert(Res == 0 && "Res is a result argument");
228 assert(ValueTy.isScalar() && "invalid offset type");
229
230 if (Value == 0) {
231 Res = Op0;
232 return std::nullopt;
233 }
234
235 Res = getMRI()->createGenericVirtualRegister(Ty: getMRI()->getType(Reg: Op0));
236 auto Cst = buildConstant(Res: ValueTy, Val: Value);
237 return buildPtrAdd(Res, Op0, Op1: Cst.getReg(Idx: 0), Flags);
238}
239
240std::optional<MachineInstrBuilder> MachineIRBuilder::materializeObjectPtrOffset(
241 Register &Res, Register Op0, const LLT ValueTy, uint64_t Value) {
242 return materializePtrAdd(Res, Op0, ValueTy, Value,
243 Flags: MachineInstr::MIFlag::NoUWrap |
244 MachineInstr::MIFlag::InBounds);
245}
246
247MachineInstrBuilder MachineIRBuilder::buildMaskLowPtrBits(const DstOp &Res,
248 const SrcOp &Op0,
249 uint32_t NumBits) {
250 LLT PtrTy = Res.getLLTTy(MRI: *getMRI());
251 LLT MaskTy = LLT::scalar(SizeInBits: PtrTy.getSizeInBits());
252 Register MaskReg = getMRI()->createGenericVirtualRegister(Ty: MaskTy);
253 buildConstant(Res: MaskReg, Val: maskTrailingZeros<uint64_t>(N: NumBits));
254 return buildPtrMask(Res, Op0, Op1: MaskReg);
255}
256
257MachineInstrBuilder
258MachineIRBuilder::buildPadVectorWithUndefElements(const DstOp &Res,
259 const SrcOp &Op0) {
260 LLT ResTy = Res.getLLTTy(MRI: *getMRI());
261 LLT Op0Ty = Op0.getLLTTy(MRI: *getMRI());
262
263 assert(ResTy.isVector() && "Res non vector type");
264
265 SmallVector<Register, 8> Regs;
266 if (Op0Ty.isVector()) {
267 assert((ResTy.getElementType() == Op0Ty.getElementType()) &&
268 "Different vector element types");
269 assert((ResTy.getNumElements() > Op0Ty.getNumElements()) &&
270 "Op0 has more elements");
271 auto Unmerge = buildUnmerge(Res: Op0Ty.getElementType(), Op: Op0);
272
273 for (auto Op : Unmerge.getInstr()->defs())
274 Regs.push_back(Elt: Op.getReg());
275 } else {
276 assert((ResTy.getSizeInBits() > Op0Ty.getSizeInBits()) &&
277 "Op0 has more size");
278 Regs.push_back(Elt: Op0.getReg());
279 }
280 Register Undef =
281 buildUndef(Res: Op0Ty.isVector() ? Op0Ty.getElementType() : Op0Ty).getReg(Idx: 0);
282 unsigned NumberOfPadElts = ResTy.getNumElements() - Regs.size();
283 for (unsigned i = 0; i < NumberOfPadElts; ++i)
284 Regs.push_back(Elt: Undef);
285 return buildMergeLikeInstr(Res, Ops: Regs);
286}
287
288MachineInstrBuilder
289MachineIRBuilder::buildDeleteTrailingVectorElements(const DstOp &Res,
290 const SrcOp &Op0) {
291 LLT ResTy = Res.getLLTTy(MRI: *getMRI());
292 LLT Op0Ty = Op0.getLLTTy(MRI: *getMRI());
293
294 assert(Op0Ty.isVector() && "Non vector type");
295 assert(((ResTy.isScalar() && (ResTy == Op0Ty.getElementType())) ||
296 (ResTy.isVector() &&
297 (ResTy.getElementType() == Op0Ty.getElementType()))) &&
298 "Different vector element types");
299 assert(
300 (ResTy.isScalar() || (ResTy.getNumElements() < Op0Ty.getNumElements())) &&
301 "Op0 has fewer elements");
302
303 auto Unmerge = buildUnmerge(Res: Op0Ty.getElementType(), Op: Op0);
304 if (ResTy.isScalar())
305 return buildCopy(Res, Op: Unmerge.getReg(Idx: 0));
306 SmallVector<Register, 8> Regs;
307 for (unsigned i = 0; i < ResTy.getNumElements(); ++i)
308 Regs.push_back(Elt: Unmerge.getReg(Idx: i));
309 return buildMergeLikeInstr(Res, Ops: Regs);
310}
311
312MachineInstrBuilder MachineIRBuilder::buildBr(MachineBasicBlock &Dest) {
313 return buildInstr(Opcode: TargetOpcode::G_BR).addMBB(MBB: &Dest);
314}
315
316MachineInstrBuilder MachineIRBuilder::buildBrIndirect(Register Tgt) {
317 assert(getMRI()->getType(Tgt).isPointer() && "invalid branch destination");
318 return buildInstr(Opcode: TargetOpcode::G_BRINDIRECT).addUse(RegNo: Tgt);
319}
320
321MachineInstrBuilder MachineIRBuilder::buildBrJT(Register TablePtr,
322 unsigned JTI,
323 Register IndexReg) {
324 assert(getMRI()->getType(TablePtr).isPointer() &&
325 "Table reg must be a pointer");
326 return buildInstr(Opcode: TargetOpcode::G_BRJT)
327 .addUse(RegNo: TablePtr)
328 .addJumpTableIndex(Idx: JTI)
329 .addUse(RegNo: IndexReg);
330}
331
332MachineInstrBuilder MachineIRBuilder::buildCopy(const DstOp &Res,
333 const SrcOp &Op) {
334 return buildInstr(Opc: TargetOpcode::COPY, DstOps: Res, SrcOps: Op);
335}
336
337MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
338 const ConstantInt &Val) {
339 assert(!isa<VectorType>(Val.getType()) && "Unexpected vector constant!");
340 LLT Ty = Res.getLLTTy(MRI: *getMRI());
341 LLT EltTy = Ty.getScalarType();
342 assert(EltTy.getScalarSizeInBits() == Val.getBitWidth() &&
343 "creating constant with the wrong size");
344
345 assert(!Ty.isScalableVector() &&
346 "unexpected scalable vector in buildConstant");
347
348 if (Ty.isFixedVector()) {
349 auto Const = buildInstr(Opcode: TargetOpcode::G_CONSTANT)
350 .addDef(RegNo: getMRI()->createGenericVirtualRegister(Ty: EltTy))
351 .addCImm(Val: &Val);
352 return buildSplatBuildVector(Res, Src: Const);
353 }
354
355 auto Const = buildInstr(Opcode: TargetOpcode::G_CONSTANT);
356 Const->setDebugLoc(DebugLoc());
357 Res.addDefToMIB(MRI&: *getMRI(), MIB&: Const);
358 Const.addCImm(Val: &Val);
359 return Const;
360}
361
362MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
363 int64_t Val) {
364 auto IntN = IntegerType::get(C&: getMF().getFunction().getContext(),
365 NumBits: Res.getLLTTy(MRI: *getMRI()).getScalarSizeInBits());
366 // TODO: Avoid implicit trunc?
367 // See https://github.com/llvm/llvm-project/issues/112510.
368 ConstantInt *CI = ConstantInt::getSigned(Ty: IntN, V: Val, /*implicitTrunc=*/ImplicitTrunc: true);
369 return buildConstant(Res, Val: *CI);
370}
371
372MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
373 const ConstantFP &Val) {
374 assert(!isa<VectorType>(Val.getType()) && "Unexpected vector constant!");
375 LLT Ty = Res.getLLTTy(MRI: *getMRI());
376 LLT EltTy = Ty.getScalarType();
377
378 assert(APFloat::getSizeInBits(Val.getValueAPF().getSemantics())
379 == EltTy.getSizeInBits() &&
380 "creating fconstant with the wrong size");
381
382 assert(!Ty.isPointer() && "invalid operand type");
383
384 assert(!Ty.isScalableVector() &&
385 "unexpected scalable vector in buildFConstant");
386
387 if (Ty.isFixedVector()) {
388 auto Const = buildInstr(Opcode: TargetOpcode::G_FCONSTANT)
389 .addDef(RegNo: getMRI()->createGenericVirtualRegister(Ty: EltTy))
390 .addFPImm(Val: &Val);
391
392 return buildSplatBuildVector(Res, Src: Const);
393 }
394
395 auto Const = buildInstr(Opcode: TargetOpcode::G_FCONSTANT);
396 Const->setDebugLoc(DebugLoc());
397 Res.addDefToMIB(MRI&: *getMRI(), MIB&: Const);
398 Const.addFPImm(Val: &Val);
399 return Const;
400}
401
402MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
403 const APInt &Val) {
404 ConstantInt *CI = ConstantInt::get(Context&: getMF().getFunction().getContext(), V: Val);
405 return buildConstant(Res, Val: *CI);
406}
407
408MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
409 double Val) {
410 LLT DstTy = Res.getLLTTy(MRI: *getMRI());
411 auto &Ctx = getMF().getFunction().getContext();
412 APFloat APF(Val);
413 bool Ignored;
414 APF.convert(ToSemantics: getFltSemanticForLLT(Ty: DstTy.getScalarType()),
415 RM: APFloat::rmNearestTiesToEven, losesInfo: &Ignored);
416 return buildFConstant(Res, Val: *ConstantFP::get(Context&: Ctx, V: APF));
417}
418
419MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
420 const APFloat &Val) {
421 auto &Ctx = getMF().getFunction().getContext();
422 auto *CFP = ConstantFP::get(Context&: Ctx, V: Val);
423 return buildFConstant(Res, Val: *CFP);
424}
425
426MachineInstrBuilder
427MachineIRBuilder::buildConstantPtrAuth(const DstOp &Res,
428 const ConstantPtrAuth *CPA,
429 Register Addr, Register AddrDisc) {
430 auto MIB = buildInstr(Opcode: TargetOpcode::G_PTRAUTH_GLOBAL_VALUE);
431 Res.addDefToMIB(MRI&: *getMRI(), MIB);
432 MIB.addUse(RegNo: Addr);
433 MIB.addImm(Val: CPA->getKey()->getZExtValue());
434 MIB.addUse(RegNo: AddrDisc);
435 MIB.addImm(Val: CPA->getDiscriminator()->getZExtValue());
436 return MIB;
437}
438
439MachineInstrBuilder MachineIRBuilder::buildBrCond(const SrcOp &Tst,
440 MachineBasicBlock &Dest) {
441 assert(Tst.getLLTTy(*getMRI()).isScalar() && "invalid operand type");
442
443 auto MIB = buildInstr(Opcode: TargetOpcode::G_BRCOND);
444 Tst.addSrcToMIB(MIB);
445 MIB.addMBB(MBB: &Dest);
446 return MIB;
447}
448
449MachineInstrBuilder
450MachineIRBuilder::buildLoad(const DstOp &Dst, const SrcOp &Addr,
451 MachinePointerInfo PtrInfo, Align Alignment,
452 MachineMemOperand::Flags MMOFlags,
453 const AAMDNodes &AAInfo) {
454 MMOFlags |= MachineMemOperand::MOLoad;
455 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
456
457 LLT Ty = Dst.getLLTTy(MRI: *getMRI());
458 MachineMemOperand *MMO =
459 getMF().getMachineMemOperand(PtrInfo, f: MMOFlags, MemTy: Ty, base_alignment: Alignment, AAInfo);
460 return buildLoad(Res: Dst, Addr, MMO&: *MMO);
461}
462
463MachineInstrBuilder MachineIRBuilder::buildLoadInstr(unsigned Opcode,
464 const DstOp &Res,
465 const SrcOp &Addr,
466 MachineMemOperand &MMO) {
467 assert(Res.getLLTTy(*getMRI()).isValid() && "invalid operand type");
468 assert(Addr.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
469
470 auto MIB = buildInstr(Opcode);
471 Res.addDefToMIB(MRI&: *getMRI(), MIB);
472 Addr.addSrcToMIB(MIB);
473 MIB.addMemOperand(MMO: &MMO);
474 return MIB;
475}
476
477MachineInstrBuilder MachineIRBuilder::buildLoadFromOffset(
478 const DstOp &Dst, const SrcOp &BasePtr,
479 MachineMemOperand &BaseMMO, int64_t Offset) {
480 LLT LoadTy = Dst.getLLTTy(MRI: *getMRI());
481 MachineMemOperand *OffsetMMO =
482 getMF().getMachineMemOperand(MMO: &BaseMMO, Offset, Ty: LoadTy);
483
484 if (Offset == 0) // This may be a size or type changing load.
485 return buildLoad(Res: Dst, Addr: BasePtr, MMO&: *OffsetMMO);
486
487 LLT PtrTy = BasePtr.getLLTTy(MRI: *getMRI());
488 LLT OffsetTy = LLT::scalar(SizeInBits: PtrTy.getSizeInBits());
489 auto ConstOffset = buildConstant(Res: OffsetTy, Val: Offset);
490 auto Ptr = buildPtrAdd(Res: PtrTy, Op0: BasePtr, Op1: ConstOffset);
491 return buildLoad(Res: Dst, Addr: Ptr, MMO&: *OffsetMMO);
492}
493
494MachineInstrBuilder MachineIRBuilder::buildStore(const SrcOp &Val,
495 const SrcOp &Addr,
496 MachineMemOperand &MMO) {
497 assert(Val.getLLTTy(*getMRI()).isValid() && "invalid operand type");
498 assert(Addr.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
499
500 auto MIB = buildInstr(Opcode: TargetOpcode::G_STORE);
501 Val.addSrcToMIB(MIB);
502 Addr.addSrcToMIB(MIB);
503 MIB.addMemOperand(MMO: &MMO);
504 return MIB;
505}
506
507MachineInstrBuilder MachineIRBuilder::buildStoreInstr(unsigned Opcode,
508 const SrcOp &Val,
509 const SrcOp &Addr,
510 MachineMemOperand &MMO) {
511 assert(Val.getLLTTy(*getMRI()).isValid() && "invalid operand type");
512 assert(Addr.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
513
514 auto MIB = buildInstr(Opcode);
515 Val.addSrcToMIB(MIB);
516 Addr.addSrcToMIB(MIB);
517 MIB.addMemOperand(MMO: &MMO);
518 return MIB;
519}
520
521MachineInstrBuilder
522MachineIRBuilder::buildStore(const SrcOp &Val, const SrcOp &Addr,
523 MachinePointerInfo PtrInfo, Align Alignment,
524 MachineMemOperand::Flags MMOFlags,
525 const AAMDNodes &AAInfo) {
526 MMOFlags |= MachineMemOperand::MOStore;
527 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
528
529 LLT Ty = Val.getLLTTy(MRI: *getMRI());
530 MachineMemOperand *MMO =
531 getMF().getMachineMemOperand(PtrInfo, f: MMOFlags, MemTy: Ty, base_alignment: Alignment, AAInfo);
532 return buildStore(Val, Addr, MMO&: *MMO);
533}
534
535MachineInstrBuilder MachineIRBuilder::buildAnyExt(const DstOp &Res,
536 const SrcOp &Op) {
537 return buildInstr(Opc: TargetOpcode::G_ANYEXT, DstOps: Res, SrcOps: Op);
538}
539
540MachineInstrBuilder MachineIRBuilder::buildSExt(const DstOp &Res,
541 const SrcOp &Op) {
542 return buildInstr(Opc: TargetOpcode::G_SEXT, DstOps: Res, SrcOps: Op);
543}
544
545MachineInstrBuilder MachineIRBuilder::buildZExt(const DstOp &Res,
546 const SrcOp &Op,
547 std::optional<unsigned> Flags) {
548 return buildInstr(Opc: TargetOpcode::G_ZEXT, DstOps: Res, SrcOps: Op, Flags);
549}
550
551unsigned MachineIRBuilder::getBoolExtOp(bool IsVec, bool IsFP) const {
552 const auto *TLI = getMF().getSubtarget().getTargetLowering();
553 switch (TLI->getBooleanContents(isVec: IsVec, isFloat: IsFP)) {
554 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
555 return TargetOpcode::G_SEXT;
556 case TargetLoweringBase::ZeroOrOneBooleanContent:
557 return TargetOpcode::G_ZEXT;
558 default:
559 return TargetOpcode::G_ANYEXT;
560 }
561}
562
563MachineInstrBuilder MachineIRBuilder::buildBoolExt(const DstOp &Res,
564 const SrcOp &Op,
565 bool IsFP) {
566 unsigned ExtOp = getBoolExtOp(IsVec: getMRI()->getType(Reg: Op.getReg()).isVector(), IsFP);
567 return buildInstr(Opc: ExtOp, DstOps: Res, SrcOps: Op);
568}
569
570MachineInstrBuilder MachineIRBuilder::buildBoolExtInReg(const DstOp &Res,
571 const SrcOp &Op,
572 bool IsVector,
573 bool IsFP) {
574 const auto *TLI = getMF().getSubtarget().getTargetLowering();
575 switch (TLI->getBooleanContents(isVec: IsVector, isFloat: IsFP)) {
576 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
577 return buildSExtInReg(Res, Op, ImmOp: 1);
578 case TargetLoweringBase::ZeroOrOneBooleanContent:
579 return buildZExtInReg(Res, Op, ImmOp: 1);
580 case TargetLoweringBase::UndefinedBooleanContent:
581 return buildCopy(Res, Op);
582 }
583
584 llvm_unreachable("unexpected BooleanContent");
585}
586
587MachineInstrBuilder MachineIRBuilder::buildExtOrTrunc(unsigned ExtOpc,
588 const DstOp &Res,
589 const SrcOp &Op) {
590 assert((TargetOpcode::G_ANYEXT == ExtOpc || TargetOpcode::G_ZEXT == ExtOpc ||
591 TargetOpcode::G_SEXT == ExtOpc) &&
592 "Expecting Extending Opc");
593 assert(Res.getLLTTy(*getMRI()).isScalar() ||
594 Res.getLLTTy(*getMRI()).isVector());
595 assert(Res.getLLTTy(*getMRI()).isScalar() ==
596 Op.getLLTTy(*getMRI()).isScalar());
597
598 unsigned Opcode = TargetOpcode::COPY;
599 if (Res.getLLTTy(MRI: *getMRI()).getSizeInBits() >
600 Op.getLLTTy(MRI: *getMRI()).getSizeInBits())
601 Opcode = ExtOpc;
602 else if (Res.getLLTTy(MRI: *getMRI()).getSizeInBits() <
603 Op.getLLTTy(MRI: *getMRI()).getSizeInBits())
604 Opcode = TargetOpcode::G_TRUNC;
605 else
606 assert(Res.getLLTTy(*getMRI()).getSizeInBits() ==
607 Op.getLLTTy(*getMRI()).getSizeInBits());
608
609 return buildInstr(Opc: Opcode, DstOps: Res, SrcOps: Op);
610}
611
612MachineInstrBuilder MachineIRBuilder::buildSExtOrTrunc(const DstOp &Res,
613 const SrcOp &Op) {
614 return buildExtOrTrunc(ExtOpc: TargetOpcode::G_SEXT, Res, Op);
615}
616
617MachineInstrBuilder MachineIRBuilder::buildZExtOrTrunc(const DstOp &Res,
618 const SrcOp &Op) {
619 return buildExtOrTrunc(ExtOpc: TargetOpcode::G_ZEXT, Res, Op);
620}
621
622MachineInstrBuilder MachineIRBuilder::buildAnyExtOrTrunc(const DstOp &Res,
623 const SrcOp &Op) {
624 return buildExtOrTrunc(ExtOpc: TargetOpcode::G_ANYEXT, Res, Op);
625}
626
627MachineInstrBuilder MachineIRBuilder::buildZExtInReg(const DstOp &Res,
628 const SrcOp &Op,
629 int64_t ImmOp) {
630 LLT ResTy = Res.getLLTTy(MRI: *getMRI());
631 auto Mask = buildConstant(
632 Res: ResTy, Val: APInt::getLowBitsSet(numBits: ResTy.getScalarSizeInBits(), loBitsSet: ImmOp));
633 return buildAnd(Dst: Res, Src0: Op, Src1: Mask);
634}
635
636MachineInstrBuilder MachineIRBuilder::buildCast(const DstOp &Dst,
637 const SrcOp &Src) {
638 LLT SrcTy = Src.getLLTTy(MRI: *getMRI());
639 LLT DstTy = Dst.getLLTTy(MRI: *getMRI());
640 if (SrcTy == DstTy)
641 return buildCopy(Res: Dst, Op: Src);
642
643 unsigned Opcode;
644 if (SrcTy.isPointerOrPointerVector())
645 Opcode = TargetOpcode::G_PTRTOINT;
646 else if (DstTy.isPointerOrPointerVector())
647 Opcode = TargetOpcode::G_INTTOPTR;
648 else {
649 assert(!SrcTy.isPointerOrPointerVector() &&
650 !DstTy.isPointerOrPointerVector() && "no G_ADDRCAST yet");
651 Opcode = TargetOpcode::G_BITCAST;
652 }
653
654 return buildInstr(Opc: Opcode, DstOps: Dst, SrcOps: Src);
655}
656
657MachineInstrBuilder MachineIRBuilder::buildExtract(const DstOp &Dst,
658 const SrcOp &Src,
659 uint64_t Index) {
660 LLT SrcTy = Src.getLLTTy(MRI: *getMRI());
661 LLT DstTy = Dst.getLLTTy(MRI: *getMRI());
662
663#ifndef NDEBUG
664 assert(SrcTy.isValid() && "invalid operand type");
665 assert(DstTy.isValid() && "invalid operand type");
666 assert(Index + DstTy.getSizeInBits() <= SrcTy.getSizeInBits() &&
667 "extracting off end of register");
668#endif
669
670 if (DstTy.getSizeInBits() == SrcTy.getSizeInBits()) {
671 assert(Index == 0 && "insertion past the end of a register");
672 return buildCast(Dst, Src);
673 }
674
675 auto Extract = buildInstr(Opcode: TargetOpcode::G_EXTRACT);
676 Dst.addDefToMIB(MRI&: *getMRI(), MIB&: Extract);
677 Src.addSrcToMIB(MIB&: Extract);
678 Extract.addImm(Val: Index);
679 return Extract;
680}
681
682MachineInstrBuilder MachineIRBuilder::buildUndef(const DstOp &Res) {
683 return buildInstr(Opc: TargetOpcode::G_IMPLICIT_DEF, DstOps: {Res}, SrcOps: {});
684}
685
686MachineInstrBuilder MachineIRBuilder::buildMergeValues(const DstOp &Res,
687 ArrayRef<Register> Ops) {
688 // Unfortunately to convert from ArrayRef<LLT> to ArrayRef<SrcOp>,
689 // we need some temporary storage for the DstOp objects. Here we use a
690 // sufficiently large SmallVector to not go through the heap.
691 SmallVector<SrcOp, 8> TmpVec(Ops);
692 assert(TmpVec.size() > 1);
693 return buildInstr(Opc: TargetOpcode::G_MERGE_VALUES, DstOps: Res, SrcOps: TmpVec);
694}
695
696MachineInstrBuilder
697MachineIRBuilder::buildMergeLikeInstr(const DstOp &Res,
698 ArrayRef<Register> Ops) {
699 // Unfortunately to convert from ArrayRef<LLT> to ArrayRef<SrcOp>,
700 // we need some temporary storage for the DstOp objects. Here we use a
701 // sufficiently large SmallVector to not go through the heap.
702 SmallVector<SrcOp, 8> TmpVec(Ops);
703 assert(TmpVec.size() > 1);
704 return buildInstr(Opc: getOpcodeForMerge(DstOp: Res, SrcOps: TmpVec), DstOps: Res, SrcOps: TmpVec);
705}
706
707MachineInstrBuilder
708MachineIRBuilder::buildMergeLikeInstr(const DstOp &Res,
709 std::initializer_list<SrcOp> Ops) {
710 assert(Ops.size() > 1);
711 return buildInstr(Opc: getOpcodeForMerge(DstOp: Res, SrcOps: Ops), DstOps: Res, SrcOps: Ops);
712}
713
714unsigned MachineIRBuilder::getOpcodeForMerge(const DstOp &DstOp,
715 ArrayRef<SrcOp> SrcOps) const {
716 if (DstOp.getLLTTy(MRI: *getMRI()).isVector()) {
717 if (SrcOps[0].getLLTTy(MRI: *getMRI()).isVector())
718 return TargetOpcode::G_CONCAT_VECTORS;
719 return TargetOpcode::G_BUILD_VECTOR;
720 }
721
722 return TargetOpcode::G_MERGE_VALUES;
723}
724
725MachineInstrBuilder MachineIRBuilder::buildUnmerge(ArrayRef<LLT> Res,
726 const SrcOp &Op) {
727 // Unfortunately to convert from ArrayRef<LLT> to ArrayRef<DstOp>,
728 // we need some temporary storage for the DstOp objects. Here we use a
729 // sufficiently large SmallVector to not go through the heap.
730 SmallVector<DstOp, 8> TmpVec(Res);
731 assert(TmpVec.size() > 1);
732 return buildInstr(Opc: TargetOpcode::G_UNMERGE_VALUES, DstOps: TmpVec, SrcOps: Op);
733}
734
735MachineInstrBuilder MachineIRBuilder::buildUnmerge(LLT Res,
736 const SrcOp &Op) {
737 unsigned NumReg = Op.getLLTTy(MRI: *getMRI()).getSizeInBits() / Res.getSizeInBits();
738 SmallVector<DstOp, 8> TmpVec(NumReg, Res);
739 return buildInstr(Opc: TargetOpcode::G_UNMERGE_VALUES, DstOps: TmpVec, SrcOps: Op);
740}
741
742MachineInstrBuilder
743MachineIRBuilder::buildUnmerge(MachineRegisterInfo::VRegAttrs Attrs,
744 const SrcOp &Op) {
745 LLT OpTy = Op.getLLTTy(MRI: *getMRI());
746 unsigned NumRegs = OpTy.getSizeInBits() / Attrs.Ty.getSizeInBits();
747 SmallVector<DstOp, 8> TmpVec(NumRegs, Attrs);
748 return buildInstr(Opc: TargetOpcode::G_UNMERGE_VALUES, DstOps: TmpVec, SrcOps: Op);
749}
750
751MachineInstrBuilder MachineIRBuilder::buildUnmerge(ArrayRef<Register> Res,
752 const SrcOp &Op) {
753 // Unfortunately to convert from ArrayRef<Register> to ArrayRef<DstOp>,
754 // we need some temporary storage for the DstOp objects. Here we use a
755 // sufficiently large SmallVector to not go through the heap.
756 SmallVector<DstOp, 8> TmpVec(Res);
757 assert(TmpVec.size() > 1);
758 return buildInstr(Opc: TargetOpcode::G_UNMERGE_VALUES, DstOps: TmpVec, SrcOps: Op);
759}
760
761MachineInstrBuilder MachineIRBuilder::buildBuildVector(const DstOp &Res,
762 ArrayRef<Register> Ops) {
763 // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
764 // we need some temporary storage for the DstOp objects. Here we use a
765 // sufficiently large SmallVector to not go through the heap.
766 SmallVector<SrcOp, 8> TmpVec(Ops);
767 return buildInstr(Opc: TargetOpcode::G_BUILD_VECTOR, DstOps: Res, SrcOps: TmpVec);
768}
769
770MachineInstrBuilder
771MachineIRBuilder::buildBuildVectorConstant(const DstOp &Res,
772 ArrayRef<APInt> Ops) {
773 SmallVector<SrcOp> TmpVec;
774 TmpVec.reserve(N: Ops.size());
775 LLT EltTy = Res.getLLTTy(MRI: *getMRI()).getElementType();
776 for (const auto &Op : Ops)
777 TmpVec.push_back(Elt: buildConstant(Res: EltTy, Val: Op));
778 return buildInstr(Opc: TargetOpcode::G_BUILD_VECTOR, DstOps: Res, SrcOps: TmpVec);
779}
780
781MachineInstrBuilder MachineIRBuilder::buildSplatBuildVector(const DstOp &Res,
782 const SrcOp &Src) {
783 SmallVector<SrcOp, 8> TmpVec(Res.getLLTTy(MRI: *getMRI()).getNumElements(), Src);
784 return buildInstr(Opc: TargetOpcode::G_BUILD_VECTOR, DstOps: Res, SrcOps: TmpVec);
785}
786
787MachineInstrBuilder
788MachineIRBuilder::buildBuildVectorTrunc(const DstOp &Res,
789 ArrayRef<Register> Ops) {
790 // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
791 // we need some temporary storage for the DstOp objects. Here we use a
792 // sufficiently large SmallVector to not go through the heap.
793 SmallVector<SrcOp, 8> TmpVec(Ops);
794 if (TmpVec[0].getLLTTy(MRI: *getMRI()).getSizeInBits() ==
795 Res.getLLTTy(MRI: *getMRI()).getElementType().getSizeInBits())
796 return buildInstr(Opc: TargetOpcode::G_BUILD_VECTOR, DstOps: Res, SrcOps: TmpVec);
797 return buildInstr(Opc: TargetOpcode::G_BUILD_VECTOR_TRUNC, DstOps: Res, SrcOps: TmpVec);
798}
799
800MachineInstrBuilder MachineIRBuilder::buildShuffleSplat(const DstOp &Res,
801 const SrcOp &Src) {
802 LLT DstTy = Res.getLLTTy(MRI: *getMRI());
803 assert(Src.getLLTTy(*getMRI()) == DstTy.getElementType() &&
804 "Expected Src to match Dst elt ty");
805 auto UndefVec = buildUndef(Res: DstTy);
806 auto Zero = buildConstant(Res: LLT::integer(SizeInBits: 64), Val: 0);
807 auto InsElt = buildInsertVectorElement(Res: DstTy, Val: UndefVec, Elt: Src, Idx: Zero);
808 SmallVector<int, 16> ZeroMask(DstTy.getNumElements());
809 return buildShuffleVector(Res: DstTy, Src1: InsElt, Src2: UndefVec, Mask: ZeroMask);
810}
811
812MachineInstrBuilder MachineIRBuilder::buildSplatVector(const DstOp &Res,
813 const SrcOp &Src) {
814 assert(Src.getLLTTy(*getMRI()) == Res.getLLTTy(*getMRI()).getElementType() &&
815 "Expected Src to match Dst elt ty");
816 return buildInstr(Opc: TargetOpcode::G_SPLAT_VECTOR, DstOps: Res, SrcOps: Src);
817}
818
819MachineInstrBuilder MachineIRBuilder::buildShuffleVector(const DstOp &Res,
820 const SrcOp &Src1,
821 const SrcOp &Src2,
822 ArrayRef<int> Mask) {
823 LLT DstTy = Res.getLLTTy(MRI: *getMRI());
824 LLT Src1Ty = Src1.getLLTTy(MRI: *getMRI());
825 LLT Src2Ty = Src2.getLLTTy(MRI: *getMRI());
826 const LLT DstElemTy = DstTy.getScalarType();
827 const LLT ElemTy1 = Src1Ty.getScalarType();
828 const LLT ElemTy2 = Src2Ty.getScalarType();
829 assert(DstElemTy == ElemTy1 && DstElemTy == ElemTy2);
830 assert(Mask.size() > 1 && "Scalar G_SHUFFLE_VECTOR are not supported");
831 (void)DstElemTy;
832 (void)ElemTy1;
833 (void)ElemTy2;
834 ArrayRef<int> MaskAlloc = getMF().allocateShuffleMask(Mask);
835 return buildInstr(Opc: TargetOpcode::G_SHUFFLE_VECTOR, DstOps: {Res}, SrcOps: {Src1, Src2})
836 .addShuffleMask(Val: MaskAlloc);
837}
838
839MachineInstrBuilder
840MachineIRBuilder::buildConcatVectors(const DstOp &Res, ArrayRef<Register> Ops) {
841 // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
842 // we need some temporary storage for the DstOp objects. Here we use a
843 // sufficiently large SmallVector to not go through the heap.
844 SmallVector<SrcOp, 8> TmpVec(Ops);
845 return buildInstr(Opc: TargetOpcode::G_CONCAT_VECTORS, DstOps: Res, SrcOps: TmpVec);
846}
847
848MachineInstrBuilder MachineIRBuilder::buildInsert(const DstOp &Res,
849 const SrcOp &Src,
850 const SrcOp &Op,
851 unsigned Index) {
852 assert(Index + Op.getLLTTy(*getMRI()).getSizeInBits() <=
853 Res.getLLTTy(*getMRI()).getSizeInBits() &&
854 "insertion past the end of a register");
855
856 if (Res.getLLTTy(MRI: *getMRI()).getSizeInBits() ==
857 Op.getLLTTy(MRI: *getMRI()).getSizeInBits()) {
858 return buildCast(Dst: Res, Src: Op);
859 }
860
861 return buildInstr(Opc: TargetOpcode::G_INSERT, DstOps: Res, SrcOps: {Src, Op, uint64_t(Index)});
862}
863
864MachineInstrBuilder MachineIRBuilder::buildStepVector(const DstOp &Res,
865 unsigned Step) {
866 unsigned Bitwidth = Res.getLLTTy(MRI: *getMRI()).getElementType().getSizeInBits();
867 ConstantInt *CI = ConstantInt::get(Context&: getMF().getFunction().getContext(),
868 V: APInt(Bitwidth, Step));
869 auto StepVector = buildInstr(Opcode: TargetOpcode::G_STEP_VECTOR);
870 StepVector->setDebugLoc(DebugLoc());
871 Res.addDefToMIB(MRI&: *getMRI(), MIB&: StepVector);
872 StepVector.addCImm(Val: CI);
873 return StepVector;
874}
875
876MachineInstrBuilder MachineIRBuilder::buildVScale(const DstOp &Res,
877 unsigned MinElts) {
878
879 auto IntN = IntegerType::get(C&: getMF().getFunction().getContext(),
880 NumBits: Res.getLLTTy(MRI: *getMRI()).getScalarSizeInBits());
881 ConstantInt *CI = ConstantInt::get(Ty: IntN, V: MinElts);
882 return buildVScale(Res, MinElts: *CI);
883}
884
885MachineInstrBuilder MachineIRBuilder::buildVScale(const DstOp &Res,
886 const ConstantInt &MinElts) {
887 auto VScale = buildInstr(Opcode: TargetOpcode::G_VSCALE);
888 VScale->setDebugLoc(DebugLoc());
889 Res.addDefToMIB(MRI&: *getMRI(), MIB&: VScale);
890 VScale.addCImm(Val: &MinElts);
891 return VScale;
892}
893
894MachineInstrBuilder MachineIRBuilder::buildVScale(const DstOp &Res,
895 const APInt &MinElts) {
896 ConstantInt *CI =
897 ConstantInt::get(Context&: getMF().getFunction().getContext(), V: MinElts);
898 return buildVScale(Res, MinElts: *CI);
899}
900
901static unsigned getIntrinsicOpcode(bool HasSideEffects, bool IsConvergent) {
902 if (HasSideEffects && IsConvergent)
903 return TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS;
904 if (HasSideEffects)
905 return TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS;
906 if (IsConvergent)
907 return TargetOpcode::G_INTRINSIC_CONVERGENT;
908 return TargetOpcode::G_INTRINSIC;
909}
910
911MachineInstrBuilder
912MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
913 ArrayRef<Register> ResultRegs,
914 bool HasSideEffects, bool isConvergent) {
915 auto MIB = buildInstr(Opcode: getIntrinsicOpcode(HasSideEffects, IsConvergent: isConvergent));
916 for (Register ResultReg : ResultRegs)
917 MIB.addDef(RegNo: ResultReg);
918 MIB.addIntrinsicID(ID);
919 return MIB;
920}
921
922MachineInstrBuilder
923MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
924 ArrayRef<Register> ResultRegs) {
925 AttributeSet Attrs = Intrinsic::getFnAttributes(C&: getContext(), id: ID);
926 bool HasSideEffects = !Attrs.getMemoryEffects().doesNotAccessMemory();
927 bool isConvergent = Attrs.hasAttribute(Kind: Attribute::Convergent);
928 return buildIntrinsic(ID, ResultRegs, HasSideEffects, isConvergent);
929}
930
931MachineInstrBuilder MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
932 ArrayRef<DstOp> Results,
933 bool HasSideEffects,
934 bool isConvergent) {
935 auto MIB = buildInstr(Opcode: getIntrinsicOpcode(HasSideEffects, IsConvergent: isConvergent));
936 for (DstOp Result : Results)
937 Result.addDefToMIB(MRI&: *getMRI(), MIB);
938 MIB.addIntrinsicID(ID);
939 return MIB;
940}
941
942MachineInstrBuilder MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
943 ArrayRef<DstOp> Results) {
944 AttributeSet Attrs = Intrinsic::getFnAttributes(C&: getContext(), id: ID);
945 bool HasSideEffects = !Attrs.getMemoryEffects().doesNotAccessMemory();
946 bool isConvergent = Attrs.hasAttribute(Kind: Attribute::Convergent);
947 return buildIntrinsic(ID, Results, HasSideEffects, isConvergent);
948}
949
950MachineInstrBuilder
951MachineIRBuilder::buildTrunc(const DstOp &Res, const SrcOp &Op,
952 std::optional<unsigned> Flags) {
953 return buildInstr(Opc: TargetOpcode::G_TRUNC, DstOps: Res, SrcOps: Op, Flags);
954}
955
956MachineInstrBuilder
957MachineIRBuilder::buildFPTrunc(const DstOp &Res, const SrcOp &Op,
958 std::optional<unsigned> Flags) {
959 return buildInstr(Opc: TargetOpcode::G_FPTRUNC, DstOps: Res, SrcOps: Op, Flags);
960}
961
962MachineInstrBuilder MachineIRBuilder::buildICmp(CmpInst::Predicate Pred,
963 const DstOp &Res,
964 const SrcOp &Op0,
965 const SrcOp &Op1,
966 std::optional<unsigned> Flags) {
967 return buildInstr(Opc: TargetOpcode::G_ICMP, DstOps: Res, SrcOps: {Pred, Op0, Op1}, Flags);
968}
969
970MachineInstrBuilder MachineIRBuilder::buildFCmp(CmpInst::Predicate Pred,
971 const DstOp &Res,
972 const SrcOp &Op0,
973 const SrcOp &Op1,
974 std::optional<unsigned> Flags) {
975
976 return buildInstr(Opc: TargetOpcode::G_FCMP, DstOps: Res, SrcOps: {Pred, Op0, Op1}, Flags);
977}
978
979MachineInstrBuilder MachineIRBuilder::buildSCmp(const DstOp &Res,
980 const SrcOp &Op0,
981 const SrcOp &Op1) {
982 return buildInstr(Opc: TargetOpcode::G_SCMP, DstOps: Res, SrcOps: {Op0, Op1});
983}
984
985MachineInstrBuilder MachineIRBuilder::buildUCmp(const DstOp &Res,
986 const SrcOp &Op0,
987 const SrcOp &Op1) {
988 return buildInstr(Opc: TargetOpcode::G_UCMP, DstOps: Res, SrcOps: {Op0, Op1});
989}
990
991MachineInstrBuilder
992MachineIRBuilder::buildSelect(const DstOp &Res, const SrcOp &Tst,
993 const SrcOp &Op0, const SrcOp &Op1,
994 std::optional<unsigned> Flags) {
995
996 return buildInstr(Opc: TargetOpcode::G_SELECT, DstOps: {Res}, SrcOps: {Tst, Op0, Op1}, Flags);
997}
998
999MachineInstrBuilder MachineIRBuilder::buildInsertSubvector(const DstOp &Res,
1000 const SrcOp &Src0,
1001 const SrcOp &Src1,
1002 unsigned Idx) {
1003 return buildInstr(Opc: TargetOpcode::G_INSERT_SUBVECTOR, DstOps: Res,
1004 SrcOps: {Src0, Src1, uint64_t(Idx)});
1005}
1006
1007MachineInstrBuilder MachineIRBuilder::buildExtractSubvector(const DstOp &Res,
1008 const SrcOp &Src,
1009 unsigned Idx) {
1010 return buildInstr(Opc: TargetOpcode::G_EXTRACT_SUBVECTOR, DstOps: Res,
1011 SrcOps: {Src, uint64_t(Idx)});
1012}
1013
1014MachineInstrBuilder
1015MachineIRBuilder::buildInsertVectorElement(const DstOp &Res, const SrcOp &Val,
1016 const SrcOp &Elt, const SrcOp &Idx) {
1017 return buildInstr(Opc: TargetOpcode::G_INSERT_VECTOR_ELT, DstOps: Res, SrcOps: {Val, Elt, Idx});
1018}
1019
1020MachineInstrBuilder
1021MachineIRBuilder::buildExtractVectorElement(const DstOp &Res, const SrcOp &Val,
1022 const SrcOp &Idx) {
1023 return buildInstr(Opc: TargetOpcode::G_EXTRACT_VECTOR_ELT, DstOps: Res, SrcOps: {Val, Idx});
1024}
1025
1026MachineInstrBuilder MachineIRBuilder::buildAtomicCmpXchgWithSuccess(
1027 const DstOp &OldValRes, const DstOp &SuccessRes, const SrcOp &Addr,
1028 const SrcOp &CmpVal, const SrcOp &NewVal, MachineMemOperand &MMO) {
1029#ifndef NDEBUG
1030 LLT OldValResTy = OldValRes.getLLTTy(*getMRI());
1031 LLT SuccessResTy = SuccessRes.getLLTTy(*getMRI());
1032 LLT AddrTy = Addr.getLLTTy(*getMRI());
1033 LLT CmpValTy = CmpVal.getLLTTy(*getMRI());
1034 LLT NewValTy = NewVal.getLLTTy(*getMRI());
1035 assert(OldValResTy.isScalar() && "invalid operand type");
1036 assert(SuccessResTy.isScalar() && "invalid operand type");
1037 assert(AddrTy.isPointer() && "invalid operand type");
1038 assert(CmpValTy.isValid() && "invalid operand type");
1039 assert(NewValTy.isValid() && "invalid operand type");
1040 assert(OldValResTy == CmpValTy && "type mismatch");
1041 assert(OldValResTy == NewValTy && "type mismatch");
1042#endif
1043
1044 auto MIB = buildInstr(Opcode: TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS);
1045 OldValRes.addDefToMIB(MRI&: *getMRI(), MIB);
1046 SuccessRes.addDefToMIB(MRI&: *getMRI(), MIB);
1047 Addr.addSrcToMIB(MIB);
1048 CmpVal.addSrcToMIB(MIB);
1049 NewVal.addSrcToMIB(MIB);
1050 MIB.addMemOperand(MMO: &MMO);
1051 return MIB;
1052}
1053
1054MachineInstrBuilder
1055MachineIRBuilder::buildAtomicCmpXchg(const DstOp &OldValRes, const SrcOp &Addr,
1056 const SrcOp &CmpVal, const SrcOp &NewVal,
1057 MachineMemOperand &MMO) {
1058#ifndef NDEBUG
1059 LLT OldValResTy = OldValRes.getLLTTy(*getMRI());
1060 LLT AddrTy = Addr.getLLTTy(*getMRI());
1061 LLT CmpValTy = CmpVal.getLLTTy(*getMRI());
1062 LLT NewValTy = NewVal.getLLTTy(*getMRI());
1063 assert(OldValResTy.isScalar() && "invalid operand type");
1064 assert(AddrTy.isPointer() && "invalid operand type");
1065 assert(CmpValTy.isValid() && "invalid operand type");
1066 assert(NewValTy.isValid() && "invalid operand type");
1067 assert(OldValResTy == CmpValTy && "type mismatch");
1068 assert(OldValResTy == NewValTy && "type mismatch");
1069#endif
1070
1071 auto MIB = buildInstr(Opcode: TargetOpcode::G_ATOMIC_CMPXCHG);
1072 OldValRes.addDefToMIB(MRI&: *getMRI(), MIB);
1073 Addr.addSrcToMIB(MIB);
1074 CmpVal.addSrcToMIB(MIB);
1075 NewVal.addSrcToMIB(MIB);
1076 MIB.addMemOperand(MMO: &MMO);
1077 return MIB;
1078}
1079
1080MachineInstrBuilder MachineIRBuilder::buildAtomicRMW(
1081 unsigned Opcode, const DstOp &OldValRes,
1082 const SrcOp &Addr, const SrcOp &Val,
1083 MachineMemOperand &MMO) {
1084
1085#ifndef NDEBUG
1086 LLT OldValResTy = OldValRes.getLLTTy(*getMRI());
1087 LLT AddrTy = Addr.getLLTTy(*getMRI());
1088 LLT ValTy = Val.getLLTTy(*getMRI());
1089 assert(AddrTy.isPointer() && "invalid operand type");
1090 assert(ValTy.isValid() && "invalid operand type");
1091 assert(OldValResTy == ValTy && "type mismatch");
1092 assert(MMO.isAtomic() && "not atomic mem operand");
1093#endif
1094
1095 auto MIB = buildInstr(Opcode);
1096 OldValRes.addDefToMIB(MRI&: *getMRI(), MIB);
1097 Addr.addSrcToMIB(MIB);
1098 Val.addSrcToMIB(MIB);
1099 MIB.addMemOperand(MMO: &MMO);
1100 return MIB;
1101}
1102
1103MachineInstrBuilder
1104MachineIRBuilder::buildAtomicRMWXchg(Register OldValRes, Register Addr,
1105 Register Val, MachineMemOperand &MMO) {
1106 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_XCHG, OldValRes, Addr, Val,
1107 MMO);
1108}
1109MachineInstrBuilder
1110MachineIRBuilder::buildAtomicRMWAdd(Register OldValRes, Register Addr,
1111 Register Val, MachineMemOperand &MMO) {
1112 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_ADD, OldValRes, Addr, Val,
1113 MMO);
1114}
1115MachineInstrBuilder
1116MachineIRBuilder::buildAtomicRMWSub(Register OldValRes, Register Addr,
1117 Register Val, MachineMemOperand &MMO) {
1118 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_SUB, OldValRes, Addr, Val,
1119 MMO);
1120}
1121MachineInstrBuilder
1122MachineIRBuilder::buildAtomicRMWAnd(Register OldValRes, Register Addr,
1123 Register Val, MachineMemOperand &MMO) {
1124 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_AND, OldValRes, Addr, Val,
1125 MMO);
1126}
1127MachineInstrBuilder
1128MachineIRBuilder::buildAtomicRMWNand(Register OldValRes, Register Addr,
1129 Register Val, MachineMemOperand &MMO) {
1130 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_NAND, OldValRes, Addr, Val,
1131 MMO);
1132}
1133MachineInstrBuilder MachineIRBuilder::buildAtomicRMWOr(Register OldValRes,
1134 Register Addr,
1135 Register Val,
1136 MachineMemOperand &MMO) {
1137 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_OR, OldValRes, Addr, Val,
1138 MMO);
1139}
1140MachineInstrBuilder
1141MachineIRBuilder::buildAtomicRMWXor(Register OldValRes, Register Addr,
1142 Register Val, MachineMemOperand &MMO) {
1143 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_XOR, OldValRes, Addr, Val,
1144 MMO);
1145}
1146MachineInstrBuilder
1147MachineIRBuilder::buildAtomicRMWMax(Register OldValRes, Register Addr,
1148 Register Val, MachineMemOperand &MMO) {
1149 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_MAX, OldValRes, Addr, Val,
1150 MMO);
1151}
1152MachineInstrBuilder
1153MachineIRBuilder::buildAtomicRMWMin(Register OldValRes, Register Addr,
1154 Register Val, MachineMemOperand &MMO) {
1155 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_MIN, OldValRes, Addr, Val,
1156 MMO);
1157}
1158MachineInstrBuilder
1159MachineIRBuilder::buildAtomicRMWUmax(Register OldValRes, Register Addr,
1160 Register Val, MachineMemOperand &MMO) {
1161 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_UMAX, OldValRes, Addr, Val,
1162 MMO);
1163}
1164MachineInstrBuilder
1165MachineIRBuilder::buildAtomicRMWUmin(Register OldValRes, Register Addr,
1166 Register Val, MachineMemOperand &MMO) {
1167 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_UMIN, OldValRes, Addr, Val,
1168 MMO);
1169}
1170
1171MachineInstrBuilder
1172MachineIRBuilder::buildAtomicRMWFAdd(
1173 const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val,
1174 MachineMemOperand &MMO) {
1175 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_FADD, OldValRes, Addr, Val,
1176 MMO);
1177}
1178
1179MachineInstrBuilder
1180MachineIRBuilder::buildAtomicRMWFSub(const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val,
1181 MachineMemOperand &MMO) {
1182 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_FSUB, OldValRes, Addr, Val,
1183 MMO);
1184}
1185
1186MachineInstrBuilder
1187MachineIRBuilder::buildAtomicRMWFMax(const DstOp &OldValRes, const SrcOp &Addr,
1188 const SrcOp &Val, MachineMemOperand &MMO) {
1189 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_FMAX, OldValRes, Addr, Val,
1190 MMO);
1191}
1192
1193MachineInstrBuilder
1194MachineIRBuilder::buildAtomicRMWFMin(const DstOp &OldValRes, const SrcOp &Addr,
1195 const SrcOp &Val, MachineMemOperand &MMO) {
1196 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_FMIN, OldValRes, Addr, Val,
1197 MMO);
1198}
1199
1200MachineInstrBuilder
1201MachineIRBuilder::buildAtomicRMWFMaximum(const DstOp &OldValRes,
1202 const SrcOp &Addr, const SrcOp &Val,
1203 MachineMemOperand &MMO) {
1204 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_FMAXIMUM, OldValRes, Addr,
1205 Val, MMO);
1206}
1207
1208MachineInstrBuilder
1209MachineIRBuilder::buildAtomicRMWFMinimum(const DstOp &OldValRes,
1210 const SrcOp &Addr, const SrcOp &Val,
1211 MachineMemOperand &MMO) {
1212 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_FMINIMUM, OldValRes, Addr,
1213 Val, MMO);
1214}
1215
1216MachineInstrBuilder
1217MachineIRBuilder::buildFence(unsigned Ordering, unsigned Scope) {
1218 return buildInstr(Opcode: TargetOpcode::G_FENCE)
1219 .addImm(Val: Ordering)
1220 .addImm(Val: Scope);
1221}
1222
1223MachineInstrBuilder MachineIRBuilder::buildPrefetch(const SrcOp &Addr,
1224 unsigned RW,
1225 unsigned Locality,
1226 unsigned CacheType,
1227 MachineMemOperand &MMO) {
1228 auto MIB = buildInstr(Opcode: TargetOpcode::G_PREFETCH);
1229 Addr.addSrcToMIB(MIB);
1230 MIB.addImm(Val: RW).addImm(Val: Locality).addImm(Val: CacheType);
1231 MIB.addMemOperand(MMO: &MMO);
1232 return MIB;
1233}
1234
1235MachineInstrBuilder
1236MachineIRBuilder::buildBlockAddress(Register Res, const BlockAddress *BA) {
1237#ifndef NDEBUG
1238 assert(getMRI()->getType(Res).isPointer() && "invalid res type");
1239#endif
1240
1241 return buildInstr(Opcode: TargetOpcode::G_BLOCK_ADDR).addDef(RegNo: Res).addBlockAddress(BA);
1242}
1243
1244void MachineIRBuilder::validateTruncExt(const LLT DstTy, const LLT SrcTy,
1245 bool IsExtend) {
1246#ifndef NDEBUG
1247 if (DstTy.isVector()) {
1248 assert(SrcTy.isVector() && "mismatched cast between vector and non-vector");
1249 assert(SrcTy.getElementCount() == DstTy.getElementCount() &&
1250 "different number of elements in a trunc/ext");
1251 } else
1252 assert(DstTy.isScalar() && SrcTy.isScalar() && "invalid extend/trunc");
1253
1254 if (IsExtend)
1255 assert(TypeSize::isKnownGT(DstTy.getSizeInBits(), SrcTy.getSizeInBits()) &&
1256 "invalid narrowing extend");
1257 else
1258 assert(TypeSize::isKnownLT(DstTy.getSizeInBits(), SrcTy.getSizeInBits()) &&
1259 "invalid widening trunc");
1260#endif
1261}
1262
1263void MachineIRBuilder::validateSelectOp(const LLT ResTy, const LLT TstTy,
1264 const LLT Op0Ty, const LLT Op1Ty) {
1265#ifndef NDEBUG
1266 assert((ResTy.isScalar() || ResTy.isVector() || ResTy.isPointer()) &&
1267 "invalid operand type");
1268 assert((ResTy == Op0Ty && ResTy == Op1Ty) && "type mismatch");
1269 if (ResTy.isScalar() || ResTy.isPointer())
1270 assert(TstTy.isScalar() && "type mismatch");
1271 else
1272 assert((TstTy.isScalar() ||
1273 (TstTy.isVector() &&
1274 TstTy.getElementCount() == Op0Ty.getElementCount())) &&
1275 "type mismatch");
1276#endif
1277}
1278
1279MachineInstrBuilder
1280MachineIRBuilder::buildInstr(unsigned Opc, ArrayRef<DstOp> DstOps,
1281 ArrayRef<SrcOp> SrcOps,
1282 std::optional<unsigned> Flags) {
1283 switch (Opc) {
1284 default:
1285 break;
1286 case TargetOpcode::G_SELECT: {
1287 assert(DstOps.size() == 1 && "Invalid select");
1288 assert(SrcOps.size() == 3 && "Invalid select");
1289 validateSelectOp(
1290 ResTy: DstOps[0].getLLTTy(MRI: *getMRI()), TstTy: SrcOps[0].getLLTTy(MRI: *getMRI()),
1291 Op0Ty: SrcOps[1].getLLTTy(MRI: *getMRI()), Op1Ty: SrcOps[2].getLLTTy(MRI: *getMRI()));
1292 break;
1293 }
1294 case TargetOpcode::G_FNEG:
1295 case TargetOpcode::G_ABS:
1296 // All these are unary ops.
1297 assert(DstOps.size() == 1 && "Invalid Dst");
1298 assert(SrcOps.size() == 1 && "Invalid Srcs");
1299 validateUnaryOp(Res: DstOps[0].getLLTTy(MRI: *getMRI()),
1300 Op0: SrcOps[0].getLLTTy(MRI: *getMRI()));
1301 break;
1302 case TargetOpcode::G_ADD:
1303 case TargetOpcode::G_AND:
1304 case TargetOpcode::G_MUL:
1305 case TargetOpcode::G_OR:
1306 case TargetOpcode::G_SUB:
1307 case TargetOpcode::G_XOR:
1308 case TargetOpcode::G_UDIV:
1309 case TargetOpcode::G_SDIV:
1310 case TargetOpcode::G_UREM:
1311 case TargetOpcode::G_SREM:
1312 case TargetOpcode::G_SMIN:
1313 case TargetOpcode::G_SMAX:
1314 case TargetOpcode::G_UMIN:
1315 case TargetOpcode::G_UMAX:
1316 case TargetOpcode::G_UADDSAT:
1317 case TargetOpcode::G_SADDSAT:
1318 case TargetOpcode::G_USUBSAT:
1319 case TargetOpcode::G_SSUBSAT: {
1320 // All these are binary ops.
1321 assert(DstOps.size() == 1 && "Invalid Dst");
1322 assert(SrcOps.size() == 2 && "Invalid Srcs");
1323 validateBinaryOp(Res: DstOps[0].getLLTTy(MRI: *getMRI()),
1324 Op0: SrcOps[0].getLLTTy(MRI: *getMRI()),
1325 Op1: SrcOps[1].getLLTTy(MRI: *getMRI()));
1326 break;
1327 }
1328 case TargetOpcode::G_SHL:
1329 case TargetOpcode::G_ASHR:
1330 case TargetOpcode::G_LSHR:
1331 case TargetOpcode::G_USHLSAT:
1332 case TargetOpcode::G_SSHLSAT: {
1333 assert(DstOps.size() == 1 && "Invalid Dst");
1334 assert(SrcOps.size() == 2 && "Invalid Srcs");
1335 validateShiftOp(Res: DstOps[0].getLLTTy(MRI: *getMRI()),
1336 Op0: SrcOps[0].getLLTTy(MRI: *getMRI()),
1337 Op1: SrcOps[1].getLLTTy(MRI: *getMRI()));
1338 break;
1339 }
1340 case TargetOpcode::G_SEXT:
1341 case TargetOpcode::G_ZEXT:
1342 case TargetOpcode::G_ANYEXT:
1343 assert(DstOps.size() == 1 && "Invalid Dst");
1344 assert(SrcOps.size() == 1 && "Invalid Srcs");
1345 validateTruncExt(DstTy: DstOps[0].getLLTTy(MRI: *getMRI()),
1346 SrcTy: SrcOps[0].getLLTTy(MRI: *getMRI()), IsExtend: true);
1347 break;
1348 case TargetOpcode::G_TRUNC:
1349 case TargetOpcode::G_FPTRUNC: {
1350 assert(DstOps.size() == 1 && "Invalid Dst");
1351 assert(SrcOps.size() == 1 && "Invalid Srcs");
1352 validateTruncExt(DstTy: DstOps[0].getLLTTy(MRI: *getMRI()),
1353 SrcTy: SrcOps[0].getLLTTy(MRI: *getMRI()), IsExtend: false);
1354 break;
1355 }
1356 case TargetOpcode::G_BITCAST: {
1357 assert(DstOps.size() == 1 && "Invalid Dst");
1358 assert(SrcOps.size() == 1 && "Invalid Srcs");
1359 assert(DstOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1360 SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() && "invalid bitcast");
1361 break;
1362 }
1363 case TargetOpcode::COPY:
1364 assert(DstOps.size() == 1 && "Invalid Dst");
1365 // If the caller wants to add a subreg source it has to be done separately
1366 // so we may not have any SrcOps at this point yet.
1367 break;
1368 case TargetOpcode::G_FCMP:
1369 case TargetOpcode::G_ICMP: {
1370 assert(DstOps.size() == 1 && "Invalid Dst Operands");
1371 assert(SrcOps.size() == 3 && "Invalid Src Operands");
1372 // For F/ICMP, the first src operand is the predicate, followed by
1373 // the two comparands.
1374 assert(SrcOps[0].getSrcOpKind() == SrcOp::SrcType::Ty_Predicate &&
1375 "Expecting predicate");
1376 assert([&]() -> bool {
1377 CmpInst::Predicate Pred = SrcOps[0].getPredicate();
1378 return Opc == TargetOpcode::G_ICMP ? CmpInst::isIntPredicate(Pred)
1379 : CmpInst::isFPPredicate(Pred);
1380 }() && "Invalid predicate");
1381 assert(SrcOps[1].getLLTTy(*getMRI()) == SrcOps[2].getLLTTy(*getMRI()) &&
1382 "Type mismatch");
1383 assert([&]() -> bool {
1384 LLT Op0Ty = SrcOps[1].getLLTTy(*getMRI());
1385 LLT DstTy = DstOps[0].getLLTTy(*getMRI());
1386 if (Op0Ty.isScalar() || Op0Ty.isPointer())
1387 return DstTy.isScalar();
1388 else
1389 return DstTy.isVector() &&
1390 DstTy.getElementCount() == Op0Ty.getElementCount();
1391 }() && "Type Mismatch");
1392 break;
1393 }
1394 case TargetOpcode::G_UNMERGE_VALUES: {
1395 assert(!DstOps.empty() && "Invalid trivial sequence");
1396 assert(SrcOps.size() == 1 && "Invalid src for Unmerge");
1397 assert(llvm::all_of(DstOps,
1398 [&, this](const DstOp &Op) {
1399 return Op.getLLTTy(*getMRI()) ==
1400 DstOps[0].getLLTTy(*getMRI());
1401 }) &&
1402 "type mismatch in output list");
1403 assert((TypeSize::ScalarTy)DstOps.size() *
1404 DstOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1405 SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1406 "input operands do not cover output register");
1407 break;
1408 }
1409 case TargetOpcode::G_MERGE_VALUES: {
1410 assert(SrcOps.size() >= 2 && "invalid trivial sequence");
1411 assert(DstOps.size() == 1 && "Invalid Dst");
1412 assert(llvm::all_of(SrcOps,
1413 [&, this](const SrcOp &Op) {
1414 return Op.getLLTTy(*getMRI()) ==
1415 SrcOps[0].getLLTTy(*getMRI());
1416 }) &&
1417 "type mismatch in input list");
1418 assert((TypeSize::ScalarTy)SrcOps.size() *
1419 SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1420 DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1421 "input operands do not cover output register");
1422 assert(!DstOps[0].getLLTTy(*getMRI()).isVector() &&
1423 "vectors should be built with G_CONCAT_VECTOR or G_BUILD_VECTOR");
1424 break;
1425 }
1426 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
1427 assert(DstOps.size() == 1 && "Invalid Dst size");
1428 assert(SrcOps.size() == 2 && "Invalid Src size");
1429 assert(SrcOps[0].getLLTTy(*getMRI()).isVector() && "Invalid operand type");
1430 assert((DstOps[0].getLLTTy(*getMRI()).isScalar() ||
1431 DstOps[0].getLLTTy(*getMRI()).isPointer()) &&
1432 "Invalid operand type");
1433 assert(SrcOps[1].getLLTTy(*getMRI()).isScalar() && "Invalid operand type");
1434 assert(SrcOps[0].getLLTTy(*getMRI()).getElementType() ==
1435 DstOps[0].getLLTTy(*getMRI()) &&
1436 "Type mismatch");
1437 break;
1438 }
1439 case TargetOpcode::G_INSERT_VECTOR_ELT: {
1440 assert(DstOps.size() == 1 && "Invalid dst size");
1441 assert(SrcOps.size() == 3 && "Invalid src size");
1442 assert(DstOps[0].getLLTTy(*getMRI()).isVector() &&
1443 SrcOps[0].getLLTTy(*getMRI()).isVector() && "Invalid operand type");
1444 assert(DstOps[0].getLLTTy(*getMRI()).getElementType() ==
1445 SrcOps[1].getLLTTy(*getMRI()) &&
1446 "Type mismatch");
1447 assert(SrcOps[2].getLLTTy(*getMRI()).isScalar() && "Invalid index");
1448 assert(DstOps[0].getLLTTy(*getMRI()).getElementCount() ==
1449 SrcOps[0].getLLTTy(*getMRI()).getElementCount() &&
1450 "Type mismatch");
1451 break;
1452 }
1453 case TargetOpcode::G_INSERT_SUBVECTOR: {
1454 assert(DstOps.size() == 1 && "Invalid Dst");
1455 assert(SrcOps.size() == 3 && "Invalid Srcs");
1456 [[maybe_unused]] LLT DstTy = DstOps[0].getLLTTy(MRI: *getMRI());
1457 [[maybe_unused]] LLT BigVecTy = SrcOps[0].getLLTTy(MRI: *getMRI());
1458 [[maybe_unused]] LLT SubVecTy = SrcOps[1].getLLTTy(MRI: *getMRI());
1459 assert(DstTy == BigVecTy &&
1460 "Dest and insert subvector source types must match!");
1461 assert(DstTy.isVector() && SubVecTy.isVector() &&
1462 "Insert subvector VTs must be vectors!");
1463 assert(DstTy.getElementType() == SubVecTy.getElementType() &&
1464 "Insert subvector VTs must have the same element type!");
1465 assert((DstTy.isScalable() || !SubVecTy.isScalable()) &&
1466 "Cannot insert a scalable vector into a fixed length vector!");
1467 assert((DstTy.isScalable() != SubVecTy.isScalable() ||
1468 DstTy.getElementCount().getKnownMinValue() >=
1469 SubVecTy.getElementCount().getKnownMinValue()) &&
1470 "Insert subvector must be from smaller vector to larger vector!");
1471 assert(SrcOps[2].getSrcOpKind() == SrcOp::SrcType::Ty_Imm &&
1472 "Insert subvector index must be constant");
1473 assert((DstTy.isScalable() != SubVecTy.isScalable() ||
1474 (SubVecTy.getElementCount().getKnownMinValue() +
1475 (uint64_t)SrcOps[2].getImm()) <=
1476 DstTy.getElementCount().getKnownMinValue()) &&
1477 "Insert subvector overflow!");
1478 assert((uint64_t)SrcOps[2].getImm() %
1479 SubVecTy.getElementCount().getKnownMinValue() ==
1480 0 &&
1481 "Insert index is not a multiple of the subvector length");
1482 break;
1483 }
1484 case TargetOpcode::G_EXTRACT_SUBVECTOR: {
1485 assert(DstOps.size() == 1 && "Invalid Dst");
1486 assert(SrcOps.size() == 2 && "Invalid Srcs");
1487 [[maybe_unused]] LLT DstTy = DstOps[0].getLLTTy(MRI: *getMRI());
1488 [[maybe_unused]] LLT SrcVecTy = SrcOps[0].getLLTTy(MRI: *getMRI());
1489 assert(DstTy.isVector() && SrcVecTy.isVector() &&
1490 "Extract subvector VTs must be vectors!");
1491 assert(DstTy.getElementType() == SrcVecTy.getElementType() &&
1492 "Extract subvector VTs must have the same element type!");
1493 assert((!DstTy.isScalable() || SrcVecTy.isScalable()) &&
1494 "Cannot extract a scalable vector from a fixed length vector!");
1495 assert((DstTy.isScalable() != SrcVecTy.isScalable() ||
1496 DstTy.getElementCount().getKnownMinValue() <=
1497 SrcVecTy.getElementCount().getKnownMinValue()) &&
1498 "Extract subvector must be from larger vector to smaller vector!");
1499 assert(SrcOps[1].getSrcOpKind() == SrcOp::SrcType::Ty_Imm &&
1500 "Extract subvector index must be a constant");
1501 assert((DstTy.isScalable() != SrcVecTy.isScalable() ||
1502 (DstTy.getElementCount().getKnownMinValue() +
1503 (uint64_t)SrcOps[1].getImm()) <=
1504 SrcVecTy.getElementCount().getKnownMinValue()) &&
1505 "Extract subvector overflow!");
1506 assert((uint64_t)SrcOps[1].getImm() %
1507 DstTy.getElementCount().getKnownMinValue() ==
1508 0 &&
1509 "Extract index is not a multiple of the output vector length");
1510 break;
1511 }
1512 case TargetOpcode::G_BUILD_VECTOR: {
1513 assert((!SrcOps.empty() || SrcOps.size() < 2) &&
1514 "Must have at least 2 operands");
1515 assert(DstOps.size() == 1 && "Invalid DstOps");
1516 assert(DstOps[0].getLLTTy(*getMRI()).isVector() &&
1517 "Res type must be a vector");
1518 assert(llvm::all_of(SrcOps,
1519 [&, this](const SrcOp &Op) {
1520 return Op.getLLTTy(*getMRI()) ==
1521 SrcOps[0].getLLTTy(*getMRI());
1522 }) &&
1523 "type mismatch in input list");
1524 assert((TypeSize::ScalarTy)SrcOps.size() *
1525 SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1526 DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1527 "input scalars do not exactly cover the output vector register");
1528 break;
1529 }
1530 case TargetOpcode::G_BUILD_VECTOR_TRUNC: {
1531 assert((!SrcOps.empty() || SrcOps.size() < 2) &&
1532 "Must have at least 2 operands");
1533 assert(DstOps.size() == 1 && "Invalid DstOps");
1534 assert(DstOps[0].getLLTTy(*getMRI()).isVector() &&
1535 "Res type must be a vector");
1536 assert(llvm::all_of(SrcOps,
1537 [&, this](const SrcOp &Op) {
1538 return Op.getLLTTy(*getMRI()) ==
1539 SrcOps[0].getLLTTy(*getMRI());
1540 }) &&
1541 "type mismatch in input list");
1542 break;
1543 }
1544 case TargetOpcode::G_CONCAT_VECTORS: {
1545 assert(DstOps.size() == 1 && "Invalid DstOps");
1546 assert((!SrcOps.empty() || SrcOps.size() < 2) &&
1547 "Must have at least 2 operands");
1548 assert(llvm::all_of(SrcOps,
1549 [&, this](const SrcOp &Op) {
1550 return (Op.getLLTTy(*getMRI()).isVector() &&
1551 Op.getLLTTy(*getMRI()) ==
1552 SrcOps[0].getLLTTy(*getMRI()));
1553 }) &&
1554 "type mismatch in input list");
1555 assert((TypeSize::ScalarTy)SrcOps.size() *
1556 SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1557 DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1558 "input vectors do not exactly cover the output vector register");
1559 break;
1560 }
1561 case TargetOpcode::G_UADDE: {
1562 assert(DstOps.size() == 2 && "Invalid no of dst operands");
1563 assert(SrcOps.size() == 3 && "Invalid no of src operands");
1564 assert(DstOps[0].getLLTTy(*getMRI()).isScalar() && "Invalid operand");
1565 assert((DstOps[0].getLLTTy(*getMRI()) == SrcOps[0].getLLTTy(*getMRI())) &&
1566 (DstOps[0].getLLTTy(*getMRI()) == SrcOps[1].getLLTTy(*getMRI())) &&
1567 "Invalid operand");
1568 assert(DstOps[1].getLLTTy(*getMRI()).isScalar() && "Invalid operand");
1569 assert(DstOps[1].getLLTTy(*getMRI()) == SrcOps[2].getLLTTy(*getMRI()) &&
1570 "type mismatch");
1571 break;
1572 }
1573 }
1574
1575 auto MIB = buildInstr(Opcode: Opc);
1576 for (const DstOp &Op : DstOps)
1577 Op.addDefToMIB(MRI&: *getMRI(), MIB);
1578 for (const SrcOp &Op : SrcOps)
1579 Op.addSrcToMIB(MIB);
1580 if (Flags)
1581 MIB->setFlags(*Flags);
1582 return MIB;
1583}
1584