1//===-- llvm/CodeGen/GlobalISel/MachineIRBuilder.cpp - MIBuilder--*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the MachineIRBuidler class.
10//===----------------------------------------------------------------------===//
11#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
12#include "llvm/CodeGen/Analysis.h"
13#include "llvm/CodeGen/MachineFunction.h"
14#include "llvm/CodeGen/MachineInstr.h"
15#include "llvm/CodeGen/MachineInstrBuilder.h"
16#include "llvm/CodeGen/MachineRegisterInfo.h"
17#include "llvm/CodeGen/TargetInstrInfo.h"
18#include "llvm/CodeGen/TargetLowering.h"
19#include "llvm/CodeGen/TargetOpcodes.h"
20#include "llvm/CodeGen/TargetSubtargetInfo.h"
21#include "llvm/IR/DebugInfoMetadata.h"
22
23using namespace llvm;
24
25void MachineIRBuilder::setMF(MachineFunction &MF) {
26 State.MF = &MF;
27 State.MBB = nullptr;
28 State.MRI = &MF.getRegInfo();
29 State.TII = MF.getSubtarget().getInstrInfo();
30 State.DL = DebugLoc();
31 State.PCSections = nullptr;
32 State.MMRA = nullptr;
33 State.II = MachineBasicBlock::iterator();
34 State.Observer = nullptr;
35}
36
37//------------------------------------------------------------------------------
38// Build instruction variants.
39//------------------------------------------------------------------------------
40
41MachineInstrBuilder MachineIRBuilder::buildInstrNoInsert(unsigned Opcode) {
42 return BuildMI(
43 MF&: getMF(),
44 MIMD: {getDL(), getPCSections(), getMMRAMetadata(), getDeactivationSymbol()},
45 MCID: getTII().get(Opcode));
46}
47
48MachineInstrBuilder MachineIRBuilder::insertInstr(MachineInstrBuilder MIB) {
49 getMBB().insert(I: getInsertPt(), MI: MIB);
50 recordInsertion(InsertedInstr: MIB);
51 return MIB;
52}
53
54MachineInstrBuilder
55MachineIRBuilder::buildDirectDbgValue(Register Reg, const MDNode *Variable,
56 const MDNode *Expr) {
57 assert(isa<DILocalVariable>(Variable) && "not a variable");
58 assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
59 assert(
60 cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
61 "Expected inlined-at fields to agree");
62 return insertInstr(MIB: BuildMI(MF&: getMF(), DL: getDL(),
63 MCID: getTII().get(Opcode: TargetOpcode::DBG_VALUE),
64 /*IsIndirect*/ false, Reg, Variable, Expr));
65}
66
67MachineInstrBuilder
68MachineIRBuilder::buildIndirectDbgValue(Register Reg, const MDNode *Variable,
69 const MDNode *Expr) {
70 assert(isa<DILocalVariable>(Variable) && "not a variable");
71 assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
72 assert(
73 cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
74 "Expected inlined-at fields to agree");
75 return insertInstr(MIB: BuildMI(MF&: getMF(), DL: getDL(),
76 MCID: getTII().get(Opcode: TargetOpcode::DBG_VALUE),
77 /*IsIndirect*/ true, Reg, Variable, Expr));
78}
79
80MachineInstrBuilder MachineIRBuilder::buildFIDbgValue(int FI,
81 const MDNode *Variable,
82 const MDNode *Expr) {
83 assert(isa<DILocalVariable>(Variable) && "not a variable");
84 assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
85 assert(
86 cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
87 "Expected inlined-at fields to agree");
88 return insertInstr(
89 MIB: BuildMI(MF&: getMF(), MIMD: getDL(), MCID: getTII().get(Opcode: TargetOpcode::DBG_VALUE))
90 .addFrameIndex(Idx: FI)
91 .addImm(Val: 0)
92 .addMetadata(MD: Variable)
93 .addMetadata(MD: Expr));
94}
95
96MachineInstrBuilder MachineIRBuilder::buildConstDbgValue(const Constant &C,
97 const MDNode *Variable,
98 const MDNode *Expr) {
99 assert(isa<DILocalVariable>(Variable) && "not a variable");
100 assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
101 assert(
102 cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
103 "Expected inlined-at fields to agree");
104 auto MIB = BuildMI(MF&: getMF(), MIMD: getDL(), MCID: getTII().get(Opcode: TargetOpcode::DBG_VALUE));
105
106 auto *NumericConstant = [&] () -> const Constant* {
107 if (const auto *CE = dyn_cast<ConstantExpr>(Val: &C))
108 if (CE->getOpcode() == Instruction::IntToPtr)
109 return CE->getOperand(i_nocapture: 0);
110 return &C;
111 }();
112
113 bool IsIndirect = true;
114 int64_t GlobalOffset;
115 if (auto *CI = dyn_cast<ConstantInt>(Val: NumericConstant)) {
116 if (CI->getBitWidth() > 64)
117 MIB.addCImm(Val: CI);
118 else if (CI->getBitWidth() == 1)
119 MIB.addImm(Val: CI->getZExtValue());
120 else
121 MIB.addImm(Val: CI->getSExtValue());
122 } else if (auto *CFP = dyn_cast<ConstantFP>(Val: NumericConstant)) {
123 MIB.addFPImm(Val: CFP);
124 } else if (isa<ConstantPointerNull>(Val: NumericConstant)) {
125 MIB.addImm(Val: 0);
126 } else if (const GlobalValue *GV = getDescribableGlobalAddress(
127 C: NumericConstant, Offset&: GlobalOffset, MF: getMF())) {
128 // The address of a global is a direct link-time constant. A displacement
129 // from it rides along in the expression rather than in the operand.
130 MIB.addGlobalAddress(GV);
131 if (GlobalOffset) {
132 SmallVector<uint64_t, 3> Ops;
133 DIExpression::appendOffset(Ops, Offset: GlobalOffset);
134 Expr = DIExpression::appendOpsToArg(Expr: cast<DIExpression>(Val: Expr), Ops, ArgNo: 0,
135 /*StackValue=*/false);
136 }
137 IsIndirect = false;
138 } else {
139 // Insert $noreg if we didn't find a usable constant and had to drop it.
140 MIB.addReg(RegNo: Register());
141 }
142
143 // DBG_VALUE spells an indirect location with a zero immediate offset operand
144 // and a direct one with $noreg. isIndirectDebugValue() ignores the offset for
145 // a non-register location operand, but isDebugOffsetImm() does not, and
146 // several consumers ask that instead.
147 if (IsIndirect)
148 MIB.addImm(Val: 0);
149 else
150 MIB.addReg(RegNo: Register());
151 MIB.addMetadata(MD: Variable).addMetadata(MD: Expr);
152 return insertInstr(MIB);
153}
154
155MachineInstrBuilder MachineIRBuilder::buildDbgLabel(const MDNode *Label) {
156 assert(isa<DILabel>(Label) && "not a label");
157 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(State.DL) &&
158 "Expected inlined-at fields to agree");
159 auto MIB = insertInstr(
160 MIB: BuildMI(MF&: getMF(), MIMD: getDL(), MCID: getTII().get(Opcode: TargetOpcode::DBG_LABEL)));
161
162 return MIB.addMetadata(MD: Label);
163}
164
165MachineInstrBuilder MachineIRBuilder::buildDynStackAlloc(const DstOp &Res,
166 const SrcOp &Size,
167 Align Alignment) {
168 assert(Res.getLLTTy(*getMRI()).isPointer() && "expected ptr dst type");
169 auto MIB = buildInstr(Opcode: TargetOpcode::G_DYN_STACKALLOC);
170 Res.addDefToMIB(MRI&: *getMRI(), MIB);
171 Size.addSrcToMIB(MIB);
172 MIB.addImm(Val: Alignment.value());
173 return MIB;
174}
175
176MachineInstrBuilder MachineIRBuilder::buildFrameIndex(const DstOp &Res,
177 int Idx) {
178 assert(Res.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
179 auto MIB = buildInstr(Opcode: TargetOpcode::G_FRAME_INDEX);
180 Res.addDefToMIB(MRI&: *getMRI(), MIB);
181 MIB.addFrameIndex(Idx);
182 return MIB;
183}
184
185MachineInstrBuilder MachineIRBuilder::buildGlobalValue(const DstOp &Res,
186 const GlobalValue *GV) {
187 assert(Res.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
188 assert(Res.getLLTTy(*getMRI()).getAddressSpace() ==
189 GV->getType()->getAddressSpace() &&
190 "address space mismatch");
191
192 auto MIB = buildInstr(Opcode: TargetOpcode::G_GLOBAL_VALUE);
193 Res.addDefToMIB(MRI&: *getMRI(), MIB);
194 MIB.addGlobalAddress(GV);
195 return MIB;
196}
197
198MachineInstrBuilder MachineIRBuilder::buildConstantPool(const DstOp &Res,
199 unsigned Idx) {
200 assert(Res.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
201 auto MIB = buildInstr(Opcode: TargetOpcode::G_CONSTANT_POOL);
202 Res.addDefToMIB(MRI&: *getMRI(), MIB);
203 MIB.addConstantPoolIndex(Idx);
204 return MIB;
205}
206
207MachineInstrBuilder MachineIRBuilder::buildJumpTable(const LLT PtrTy,
208 unsigned JTI) {
209 return buildInstr(Opc: TargetOpcode::G_JUMP_TABLE, DstOps: {PtrTy}, SrcOps: {})
210 .addJumpTableIndex(Idx: JTI);
211}
212
213void MachineIRBuilder::validateUnaryOp(const LLT Res, const LLT Op0) {
214 assert((Res.isScalar() || Res.isVector()) && "invalid operand type");
215 assert((Res == Op0) && "type mismatch");
216}
217
218void MachineIRBuilder::validateBinaryOp(const LLT Res, const LLT Op0,
219 const LLT Op1) {
220 assert((Res.isScalar() || Res.isVector()) && "invalid operand type");
221 assert((Res == Op0 && Res == Op1) && "type mismatch");
222}
223
224void MachineIRBuilder::validateShiftOp(const LLT Res, const LLT Op0,
225 const LLT Op1) {
226 assert((Res.isScalar() || Res.isVector()) && "invalid operand type");
227 assert((Res == Op0) && "type mismatch");
228}
229
230MachineInstrBuilder
231MachineIRBuilder::buildPtrAdd(const DstOp &Res, const SrcOp &Op0,
232 const SrcOp &Op1, std::optional<unsigned> Flags) {
233 assert(Res.getLLTTy(*getMRI()).isPointerOrPointerVector() &&
234 Res.getLLTTy(*getMRI()) == Op0.getLLTTy(*getMRI()) && "type mismatch");
235 assert(Op1.getLLTTy(*getMRI()).getScalarType().isScalar() && "invalid offset type");
236
237 return buildInstr(Opc: TargetOpcode::G_PTR_ADD, DstOps: {Res}, SrcOps: {Op0, Op1}, Flags);
238}
239
240MachineInstrBuilder MachineIRBuilder::buildObjectPtrOffset(const DstOp &Res,
241 const SrcOp &Op0,
242 const SrcOp &Op1) {
243 return buildPtrAdd(Res, Op0, Op1,
244 Flags: MachineInstr::MIFlag::NoUWrap |
245 MachineInstr::MIFlag::InBounds);
246}
247
248std::optional<MachineInstrBuilder>
249MachineIRBuilder::materializePtrAdd(Register &Res, Register Op0,
250 const LLT ValueTy, uint64_t Value,
251 std::optional<unsigned> Flags) {
252 assert(Res == 0 && "Res is a result argument");
253 assert(ValueTy.isScalar() && "invalid offset type");
254
255 if (Value == 0) {
256 Res = Op0;
257 return std::nullopt;
258 }
259
260 Res = getMRI()->createGenericVirtualRegister(Ty: getMRI()->getType(Reg: Op0));
261 auto Cst = buildConstant(Res: ValueTy, Val: Value);
262 return buildPtrAdd(Res, Op0, Op1: Cst.getReg(Idx: 0), Flags);
263}
264
265std::optional<MachineInstrBuilder> MachineIRBuilder::materializeObjectPtrOffset(
266 Register &Res, Register Op0, const LLT ValueTy, uint64_t Value) {
267 return materializePtrAdd(Res, Op0, ValueTy, Value,
268 Flags: MachineInstr::MIFlag::NoUWrap |
269 MachineInstr::MIFlag::InBounds);
270}
271
272MachineInstrBuilder MachineIRBuilder::buildMaskLowPtrBits(const DstOp &Res,
273 const SrcOp &Op0,
274 uint32_t NumBits) {
275 LLT PtrTy = Res.getLLTTy(MRI: *getMRI());
276 LLT MaskTy = LLT::integer(SizeInBits: PtrTy.getSizeInBits());
277 Register MaskReg = getMRI()->createGenericVirtualRegister(Ty: MaskTy);
278 buildConstant(Res: MaskReg, Val: maskTrailingZeros<uint64_t>(N: NumBits));
279 return buildPtrMask(Res, Op0, Op1: MaskReg);
280}
281
282MachineInstrBuilder
283MachineIRBuilder::buildPadVectorWithUndefElements(const DstOp &Res,
284 const SrcOp &Op0) {
285 LLT ResTy = Res.getLLTTy(MRI: *getMRI());
286 LLT Op0Ty = Op0.getLLTTy(MRI: *getMRI());
287
288 assert(ResTy.isVector() && "Res non vector type");
289
290 SmallVector<Register, 8> Regs;
291 if (Op0Ty.isVector()) {
292 assert((ResTy.getElementType() == Op0Ty.getElementType()) &&
293 "Different vector element types");
294 assert((ResTy.getNumElements() > Op0Ty.getNumElements()) &&
295 "Op0 has more elements");
296 auto Unmerge = buildUnmerge(Res: Op0Ty.getElementType(), Op: Op0);
297
298 for (auto Op : Unmerge.getInstr()->defs())
299 Regs.push_back(Elt: Op.getReg());
300 } else {
301 assert((ResTy.getSizeInBits() > Op0Ty.getSizeInBits()) &&
302 "Op0 has more size");
303 Regs.push_back(Elt: Op0.getReg());
304 }
305 Register Undef =
306 buildUndef(Res: Op0Ty.isVector() ? Op0Ty.getElementType() : Op0Ty).getReg(Idx: 0);
307 unsigned NumberOfPadElts = ResTy.getNumElements() - Regs.size();
308 for (unsigned i = 0; i < NumberOfPadElts; ++i)
309 Regs.push_back(Elt: Undef);
310 return buildMergeLikeInstr(Res, Ops: Regs);
311}
312
313MachineInstrBuilder
314MachineIRBuilder::buildDeleteTrailingVectorElements(const DstOp &Res,
315 const SrcOp &Op0) {
316 LLT ResTy = Res.getLLTTy(MRI: *getMRI());
317 LLT Op0Ty = Op0.getLLTTy(MRI: *getMRI());
318
319 assert(Op0Ty.isVector() && "Non vector type");
320 assert(((ResTy.isScalar() && (ResTy == Op0Ty.getElementType())) ||
321 (ResTy.isVector() &&
322 (ResTy.getElementType() == Op0Ty.getElementType()))) &&
323 "Different vector element types");
324 assert(
325 (ResTy.isScalar() || (ResTy.getNumElements() < Op0Ty.getNumElements())) &&
326 "Op0 has fewer elements");
327
328 auto Unmerge = buildUnmerge(Res: Op0Ty.getElementType(), Op: Op0);
329 if (ResTy.isScalar())
330 return buildCopy(Res, Op: Unmerge.getReg(Idx: 0));
331 SmallVector<Register, 8> Regs;
332 for (unsigned i = 0; i < ResTy.getNumElements(); ++i)
333 Regs.push_back(Elt: Unmerge.getReg(Idx: i));
334 return buildMergeLikeInstr(Res, Ops: Regs);
335}
336
337MachineInstrBuilder MachineIRBuilder::buildBr(MachineBasicBlock &Dest) {
338 return buildInstr(Opcode: TargetOpcode::G_BR).addMBB(MBB: &Dest);
339}
340
341MachineInstrBuilder MachineIRBuilder::buildBrIndirect(Register Tgt) {
342 assert(getMRI()->getType(Tgt).isPointer() && "invalid branch destination");
343 return buildInstr(Opcode: TargetOpcode::G_BRINDIRECT).addUse(RegNo: Tgt);
344}
345
346MachineInstrBuilder MachineIRBuilder::buildBrJT(Register TablePtr,
347 unsigned JTI,
348 Register IndexReg) {
349 assert(getMRI()->getType(TablePtr).isPointer() &&
350 "Table reg must be a pointer");
351 return buildInstr(Opcode: TargetOpcode::G_BRJT)
352 .addUse(RegNo: TablePtr)
353 .addJumpTableIndex(Idx: JTI)
354 .addUse(RegNo: IndexReg);
355}
356
357MachineInstrBuilder MachineIRBuilder::buildCopy(const DstOp &Res,
358 const SrcOp &Op) {
359 return buildInstr(Opc: TargetOpcode::COPY, DstOps: Res, SrcOps: Op);
360}
361
362MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
363 const ConstantInt &Val) {
364 assert(!isa<VectorType>(Val.getType()) && "Unexpected vector constant!");
365 LLT Ty = Res.getLLTTy(MRI: *getMRI());
366 LLT EltTy = Ty.getScalarType();
367 assert(EltTy.getScalarSizeInBits() == Val.getBitWidth() &&
368 "creating constant with the wrong size");
369
370 assert(!Ty.isScalableVector() &&
371 "unexpected scalable vector in buildConstant");
372
373 if (Ty.isFixedVector()) {
374 auto Const = buildInstr(Opcode: TargetOpcode::G_CONSTANT)
375 .addDef(RegNo: getMRI()->createGenericVirtualRegister(Ty: EltTy))
376 .addCImm(Val: &Val);
377 return buildSplatBuildVector(Res, Src: Const);
378 }
379
380 auto Const = buildInstr(Opcode: TargetOpcode::G_CONSTANT);
381 Const->setDebugLoc(DebugLoc());
382 Res.addDefToMIB(MRI&: *getMRI(), MIB&: Const);
383 Const.addCImm(Val: &Val);
384 return Const;
385}
386
387MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
388 int64_t Val) {
389 auto IntN = IntegerType::get(C&: getMF().getFunction().getContext(),
390 NumBits: Res.getLLTTy(MRI: *getMRI()).getScalarSizeInBits());
391 // TODO: Avoid implicit trunc?
392 // See https://github.com/llvm/llvm-project/issues/112510.
393 ConstantInt *CI = ConstantInt::getSigned(Ty: IntN, V: Val, /*implicitTrunc=*/ImplicitTrunc: true);
394 return buildConstant(Res, Val: *CI);
395}
396
397MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
398 const ConstantFP &Val) {
399 assert(!isa<VectorType>(Val.getType()) && "Unexpected vector constant!");
400 LLT Ty = Res.getLLTTy(MRI: *getMRI());
401 LLT EltTy = Ty.getScalarType();
402
403 assert(APFloat::getSizeInBits(Val.getValueAPF().getSemantics())
404 == EltTy.getSizeInBits() &&
405 "creating fconstant with the wrong size");
406
407 assert(!Ty.isPointer() && "invalid operand type");
408
409 assert(!Ty.isScalableVector() &&
410 "unexpected scalable vector in buildFConstant");
411
412 if (Ty.isFixedVector()) {
413 auto Const = buildInstr(Opcode: TargetOpcode::G_FCONSTANT)
414 .addDef(RegNo: getMRI()->createGenericVirtualRegister(Ty: EltTy))
415 .addFPImm(Val: &Val);
416
417 return buildSplatBuildVector(Res, Src: Const);
418 }
419
420 auto Const = buildInstr(Opcode: TargetOpcode::G_FCONSTANT);
421 Const->setDebugLoc(DebugLoc());
422 Res.addDefToMIB(MRI&: *getMRI(), MIB&: Const);
423 Const.addFPImm(Val: &Val);
424 return Const;
425}
426
427MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
428 const APInt &Val) {
429 ConstantInt *CI = ConstantInt::get(Context&: getMF().getFunction().getContext(), V: Val);
430 return buildConstant(Res, Val: *CI);
431}
432
433MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
434 double Val) {
435 LLT DstTy = Res.getLLTTy(MRI: *getMRI());
436 auto &Ctx = getMF().getFunction().getContext();
437 APFloat APF(Val);
438 bool Ignored;
439 APF.convert(ToSemantics: getFltSemanticForLLT(Ty: DstTy.getScalarType()),
440 RM: APFloat::rmNearestTiesToEven, losesInfo: &Ignored);
441 return buildFConstant(Res, Val: *ConstantFP::get(Context&: Ctx, V: APF));
442}
443
444MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
445 const APFloat &Val) {
446 auto &Ctx = getMF().getFunction().getContext();
447 auto *CFP = ConstantFP::get(Context&: Ctx, V: Val);
448 return buildFConstant(Res, Val: *CFP);
449}
450
451MachineInstrBuilder
452MachineIRBuilder::buildConstantPtrAuth(const DstOp &Res,
453 const ConstantPtrAuth *CPA,
454 Register Addr, Register AddrDisc) {
455 auto MIB = buildInstr(Opcode: TargetOpcode::G_PTRAUTH_GLOBAL_VALUE);
456 Res.addDefToMIB(MRI&: *getMRI(), MIB);
457 MIB.addUse(RegNo: Addr);
458 MIB.addImm(Val: CPA->getKey()->getZExtValue());
459 MIB.addUse(RegNo: AddrDisc);
460 MIB.addImm(Val: CPA->getDiscriminator()->getZExtValue());
461 return MIB;
462}
463
464MachineInstrBuilder MachineIRBuilder::buildBrCond(const SrcOp &Tst,
465 MachineBasicBlock &Dest) {
466 assert(Tst.getLLTTy(*getMRI()).isScalar() && "invalid operand type");
467
468 auto MIB = buildInstr(Opcode: TargetOpcode::G_BRCOND);
469 Tst.addSrcToMIB(MIB);
470 MIB.addMBB(MBB: &Dest);
471 return MIB;
472}
473
474MachineInstrBuilder
475MachineIRBuilder::buildLoad(const DstOp &Dst, const SrcOp &Addr,
476 MachinePointerInfo PtrInfo, Align Alignment,
477 MachineMemOperand::Flags MMOFlags,
478 const AAMDNodes &AAInfo) {
479 MMOFlags |= MachineMemOperand::MOLoad;
480 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
481
482 LLT Ty = Dst.getLLTTy(MRI: *getMRI());
483 MachineMemOperand *MMO =
484 getMF().getMachineMemOperand(PtrInfo, F: MMOFlags, MemTy: Ty, BaseAlignment: Alignment, Metadata: AAInfo);
485 return buildLoad(Res: Dst, Addr, MMO&: *MMO);
486}
487
488MachineInstrBuilder MachineIRBuilder::buildLoadInstr(unsigned Opcode,
489 const DstOp &Res,
490 const SrcOp &Addr,
491 MachineMemOperand &MMO) {
492 assert(Res.getLLTTy(*getMRI()).isValid() && "invalid operand type");
493 assert(Addr.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
494
495 auto MIB = buildInstr(Opcode);
496 Res.addDefToMIB(MRI&: *getMRI(), MIB);
497 Addr.addSrcToMIB(MIB);
498 MIB.addMemOperand(MMO: &MMO);
499 return MIB;
500}
501
502MachineInstrBuilder MachineIRBuilder::buildLoadFromOffset(
503 const DstOp &Dst, const SrcOp &BasePtr,
504 MachineMemOperand &BaseMMO, int64_t Offset) {
505 LLT LoadTy = Dst.getLLTTy(MRI: *getMRI());
506 MachineMemOperand *OffsetMMO =
507 getMF().getMachineMemOperand(MMO: &BaseMMO, Offset, Ty: LoadTy);
508
509 if (Offset == 0) // This may be a size or type changing load.
510 return buildLoad(Res: Dst, Addr: BasePtr, MMO&: *OffsetMMO);
511
512 LLT PtrTy = BasePtr.getLLTTy(MRI: *getMRI());
513 LLT OffsetTy = LLT::scalar(SizeInBits: PtrTy.getSizeInBits());
514 auto ConstOffset = buildConstant(Res: OffsetTy, Val: Offset);
515 auto Ptr = buildPtrAdd(Res: PtrTy, Op0: BasePtr, Op1: ConstOffset);
516 return buildLoad(Res: Dst, Addr: Ptr, MMO&: *OffsetMMO);
517}
518
519MachineInstrBuilder MachineIRBuilder::buildStore(const SrcOp &Val,
520 const SrcOp &Addr,
521 MachineMemOperand &MMO) {
522 assert(Val.getLLTTy(*getMRI()).isValid() && "invalid operand type");
523 assert(Addr.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
524
525 auto MIB = buildInstr(Opcode: TargetOpcode::G_STORE);
526 Val.addSrcToMIB(MIB);
527 Addr.addSrcToMIB(MIB);
528 MIB.addMemOperand(MMO: &MMO);
529 return MIB;
530}
531
532MachineInstrBuilder MachineIRBuilder::buildStoreInstr(unsigned Opcode,
533 const SrcOp &Val,
534 const SrcOp &Addr,
535 MachineMemOperand &MMO) {
536 assert(Val.getLLTTy(*getMRI()).isValid() && "invalid operand type");
537 assert(Addr.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
538
539 auto MIB = buildInstr(Opcode);
540 Val.addSrcToMIB(MIB);
541 Addr.addSrcToMIB(MIB);
542 MIB.addMemOperand(MMO: &MMO);
543 return MIB;
544}
545
546MachineInstrBuilder
547MachineIRBuilder::buildStore(const SrcOp &Val, const SrcOp &Addr,
548 MachinePointerInfo PtrInfo, Align Alignment,
549 MachineMemOperand::Flags MMOFlags,
550 const AAMDNodes &AAInfo) {
551 MMOFlags |= MachineMemOperand::MOStore;
552 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
553
554 LLT Ty = Val.getLLTTy(MRI: *getMRI());
555 MachineMemOperand *MMO =
556 getMF().getMachineMemOperand(PtrInfo, F: MMOFlags, MemTy: Ty, BaseAlignment: Alignment, Metadata: AAInfo);
557 return buildStore(Val, Addr, MMO&: *MMO);
558}
559
560MachineInstrBuilder MachineIRBuilder::buildAnyExt(const DstOp &Res,
561 const SrcOp &Op) {
562 return buildInstr(Opc: TargetOpcode::G_ANYEXT, DstOps: Res, SrcOps: Op);
563}
564
565MachineInstrBuilder MachineIRBuilder::buildSExt(const DstOp &Res,
566 const SrcOp &Op) {
567 return buildInstr(Opc: TargetOpcode::G_SEXT, DstOps: Res, SrcOps: Op);
568}
569
570MachineInstrBuilder MachineIRBuilder::buildZExt(const DstOp &Res,
571 const SrcOp &Op,
572 std::optional<unsigned> Flags) {
573 return buildInstr(Opc: TargetOpcode::G_ZEXT, DstOps: Res, SrcOps: Op, Flags);
574}
575
576unsigned MachineIRBuilder::getBoolExtOp(bool IsVec, bool IsFP) const {
577 const auto *TLI = getMF().getSubtarget().getTargetLowering();
578 switch (TLI->getBooleanContents(isVec: IsVec, isFloat: IsFP)) {
579 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
580 return TargetOpcode::G_SEXT;
581 case TargetLoweringBase::ZeroOrOneBooleanContent:
582 return TargetOpcode::G_ZEXT;
583 default:
584 return TargetOpcode::G_ANYEXT;
585 }
586}
587
588MachineInstrBuilder MachineIRBuilder::buildBoolExt(const DstOp &Res,
589 const SrcOp &Op,
590 bool IsFP) {
591 unsigned ExtOp = getBoolExtOp(IsVec: getMRI()->getType(Reg: Op.getReg()).isVector(), IsFP);
592 return buildInstr(Opc: ExtOp, DstOps: Res, SrcOps: Op);
593}
594
595MachineInstrBuilder MachineIRBuilder::buildBoolExtInReg(const DstOp &Res,
596 const SrcOp &Op,
597 bool IsVector,
598 bool IsFP) {
599 const auto *TLI = getMF().getSubtarget().getTargetLowering();
600 switch (TLI->getBooleanContents(isVec: IsVector, isFloat: IsFP)) {
601 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
602 return buildSExtInReg(Res, Op, ImmOp: 1);
603 case TargetLoweringBase::ZeroOrOneBooleanContent:
604 return buildZExtInReg(Res, Op, ImmOp: 1);
605 case TargetLoweringBase::UndefinedBooleanContent:
606 return buildCopy(Res, Op);
607 }
608
609 llvm_unreachable("unexpected BooleanContent");
610}
611
612MachineInstrBuilder MachineIRBuilder::buildExtOrTrunc(unsigned ExtOpc,
613 const DstOp &Res,
614 const SrcOp &Op) {
615 assert((TargetOpcode::G_ANYEXT == ExtOpc || TargetOpcode::G_ZEXT == ExtOpc ||
616 TargetOpcode::G_SEXT == ExtOpc) &&
617 "Expecting Extending Opc");
618 assert(Res.getLLTTy(*getMRI()).isScalar() ||
619 Res.getLLTTy(*getMRI()).isVector());
620 assert(Res.getLLTTy(*getMRI()).isScalar() ==
621 Op.getLLTTy(*getMRI()).isScalar());
622
623 unsigned Opcode = TargetOpcode::COPY;
624 if (Res.getLLTTy(MRI: *getMRI()).getSizeInBits() >
625 Op.getLLTTy(MRI: *getMRI()).getSizeInBits())
626 Opcode = ExtOpc;
627 else if (Res.getLLTTy(MRI: *getMRI()).getSizeInBits() <
628 Op.getLLTTy(MRI: *getMRI()).getSizeInBits())
629 Opcode = TargetOpcode::G_TRUNC;
630 else
631 assert(Res.getLLTTy(*getMRI()).getSizeInBits() ==
632 Op.getLLTTy(*getMRI()).getSizeInBits());
633
634 return buildInstr(Opc: Opcode, DstOps: Res, SrcOps: Op);
635}
636
637MachineInstrBuilder MachineIRBuilder::buildSExtOrTrunc(const DstOp &Res,
638 const SrcOp &Op) {
639 return buildExtOrTrunc(ExtOpc: TargetOpcode::G_SEXT, Res, Op);
640}
641
642MachineInstrBuilder MachineIRBuilder::buildZExtOrTrunc(const DstOp &Res,
643 const SrcOp &Op) {
644 return buildExtOrTrunc(ExtOpc: TargetOpcode::G_ZEXT, Res, Op);
645}
646
647MachineInstrBuilder MachineIRBuilder::buildAnyExtOrTrunc(const DstOp &Res,
648 const SrcOp &Op) {
649 return buildExtOrTrunc(ExtOpc: TargetOpcode::G_ANYEXT, Res, Op);
650}
651
652MachineInstrBuilder MachineIRBuilder::buildZExtInReg(const DstOp &Res,
653 const SrcOp &Op,
654 int64_t ImmOp) {
655 LLT ResTy = Res.getLLTTy(MRI: *getMRI());
656 auto Mask = buildConstant(
657 Res: ResTy, Val: APInt::getLowBitsSet(numBits: ResTy.getScalarSizeInBits(), loBitsSet: ImmOp));
658 return buildAnd(Dst: Res, Src0: Op, Src1: Mask);
659}
660
661MachineInstrBuilder MachineIRBuilder::buildCast(const DstOp &Dst,
662 const SrcOp &Src) {
663 LLT SrcTy = Src.getLLTTy(MRI: *getMRI());
664 LLT DstTy = Dst.getLLTTy(MRI: *getMRI());
665 if (SrcTy == DstTy)
666 return buildCopy(Res: Dst, Op: Src);
667
668 unsigned Opcode;
669 if (SrcTy.isPointerOrPointerVector())
670 Opcode = TargetOpcode::G_PTRTOINT;
671 else if (DstTy.isPointerOrPointerVector())
672 Opcode = TargetOpcode::G_INTTOPTR;
673 else {
674 assert(!SrcTy.isPointerOrPointerVector() &&
675 !DstTy.isPointerOrPointerVector() && "no G_ADDRCAST yet");
676 Opcode = TargetOpcode::G_BITCAST;
677 }
678
679 return buildInstr(Opc: Opcode, DstOps: Dst, SrcOps: Src);
680}
681
682MachineInstrBuilder MachineIRBuilder::buildExtract(const DstOp &Dst,
683 const SrcOp &Src,
684 uint64_t Index) {
685 LLT SrcTy = Src.getLLTTy(MRI: *getMRI());
686 LLT DstTy = Dst.getLLTTy(MRI: *getMRI());
687
688#ifndef NDEBUG
689 assert(SrcTy.isValid() && "invalid operand type");
690 assert(DstTy.isValid() && "invalid operand type");
691 assert(Index + DstTy.getSizeInBits() <= SrcTy.getSizeInBits() &&
692 "extracting off end of register");
693#endif
694
695 if (DstTy.getSizeInBits() == SrcTy.getSizeInBits()) {
696 assert(Index == 0 && "insertion past the end of a register");
697 return buildCast(Dst, Src);
698 }
699
700 auto Extract = buildInstr(Opcode: TargetOpcode::G_EXTRACT);
701 Dst.addDefToMIB(MRI&: *getMRI(), MIB&: Extract);
702 Src.addSrcToMIB(MIB&: Extract);
703 Extract.addImm(Val: Index);
704 return Extract;
705}
706
707MachineInstrBuilder MachineIRBuilder::buildUndef(const DstOp &Res) {
708 return buildInstr(Opc: TargetOpcode::G_IMPLICIT_DEF, DstOps: {Res}, SrcOps: {});
709}
710
711MachineInstrBuilder MachineIRBuilder::buildMergeValues(const DstOp &Res,
712 ArrayRef<Register> Ops) {
713 // Unfortunately to convert from ArrayRef<LLT> to ArrayRef<SrcOp>,
714 // we need some temporary storage for the DstOp objects. Here we use a
715 // sufficiently large SmallVector to not go through the heap.
716 SmallVector<SrcOp, 8> TmpVec(Ops);
717 assert(TmpVec.size() > 1);
718 return buildInstr(Opc: TargetOpcode::G_MERGE_VALUES, DstOps: Res, SrcOps: TmpVec);
719}
720
721MachineInstrBuilder
722MachineIRBuilder::buildMergeLikeInstr(const DstOp &Res,
723 ArrayRef<Register> Ops) {
724 // Unfortunately to convert from ArrayRef<LLT> to ArrayRef<SrcOp>,
725 // we need some temporary storage for the DstOp objects. Here we use a
726 // sufficiently large SmallVector to not go through the heap.
727 SmallVector<SrcOp, 8> TmpVec(Ops);
728 assert(TmpVec.size() > 1);
729 return buildInstr(Opc: getOpcodeForMerge(DstOp: Res, SrcOps: TmpVec), DstOps: Res, SrcOps: TmpVec);
730}
731
732MachineInstrBuilder
733MachineIRBuilder::buildMergeLikeInstr(const DstOp &Res,
734 std::initializer_list<SrcOp> Ops) {
735 assert(Ops.size() > 1);
736 return buildInstr(Opc: getOpcodeForMerge(DstOp: Res, SrcOps: Ops), DstOps: Res, SrcOps: Ops);
737}
738
739unsigned MachineIRBuilder::getOpcodeForMerge(const DstOp &DstOp,
740 ArrayRef<SrcOp> SrcOps) const {
741 if (DstOp.getLLTTy(MRI: *getMRI()).isVector()) {
742 if (SrcOps[0].getLLTTy(MRI: *getMRI()).isVector())
743 return TargetOpcode::G_CONCAT_VECTORS;
744 return TargetOpcode::G_BUILD_VECTOR;
745 }
746
747 return TargetOpcode::G_MERGE_VALUES;
748}
749
750MachineInstrBuilder MachineIRBuilder::buildUnmerge(ArrayRef<LLT> Res,
751 const SrcOp &Op) {
752 // Unfortunately to convert from ArrayRef<LLT> to ArrayRef<DstOp>,
753 // we need some temporary storage for the DstOp objects. Here we use a
754 // sufficiently large SmallVector to not go through the heap.
755 SmallVector<DstOp, 8> TmpVec(Res);
756 assert(TmpVec.size() > 1);
757 return buildInstr(Opc: TargetOpcode::G_UNMERGE_VALUES, DstOps: TmpVec, SrcOps: Op);
758}
759
760MachineInstrBuilder MachineIRBuilder::buildUnmerge(LLT Res,
761 const SrcOp &Op) {
762 unsigned NumReg = Op.getLLTTy(MRI: *getMRI()).getSizeInBits() / Res.getSizeInBits();
763 SmallVector<DstOp, 8> TmpVec(NumReg, Res);
764 return buildInstr(Opc: TargetOpcode::G_UNMERGE_VALUES, DstOps: TmpVec, SrcOps: Op);
765}
766
767MachineInstrBuilder
768MachineIRBuilder::buildUnmerge(MachineRegisterInfo::VRegAttrs Attrs,
769 const SrcOp &Op) {
770 LLT OpTy = Op.getLLTTy(MRI: *getMRI());
771 unsigned NumRegs = OpTy.getSizeInBits() / Attrs.Ty.getSizeInBits();
772 SmallVector<DstOp, 8> TmpVec(NumRegs, Attrs);
773 return buildInstr(Opc: TargetOpcode::G_UNMERGE_VALUES, DstOps: TmpVec, SrcOps: Op);
774}
775
776MachineInstrBuilder MachineIRBuilder::buildUnmerge(ArrayRef<Register> Res,
777 const SrcOp &Op) {
778 // Unfortunately to convert from ArrayRef<Register> to ArrayRef<DstOp>,
779 // we need some temporary storage for the DstOp objects. Here we use a
780 // sufficiently large SmallVector to not go through the heap.
781 SmallVector<DstOp, 8> TmpVec(Res);
782 assert(TmpVec.size() > 1);
783 return buildInstr(Opc: TargetOpcode::G_UNMERGE_VALUES, DstOps: TmpVec, SrcOps: Op);
784}
785
786MachineInstrBuilder MachineIRBuilder::buildBuildVector(const DstOp &Res,
787 ArrayRef<Register> Ops) {
788 // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
789 // we need some temporary storage for the DstOp objects. Here we use a
790 // sufficiently large SmallVector to not go through the heap.
791 SmallVector<SrcOp, 8> TmpVec(Ops);
792 return buildInstr(Opc: TargetOpcode::G_BUILD_VECTOR, DstOps: Res, SrcOps: TmpVec);
793}
794
795MachineInstrBuilder
796MachineIRBuilder::buildBuildVectorConstant(const DstOp &Res,
797 ArrayRef<APInt> Ops) {
798 SmallVector<SrcOp> TmpVec;
799 TmpVec.reserve(N: Ops.size());
800 LLT EltTy = Res.getLLTTy(MRI: *getMRI()).getElementType();
801 for (const auto &Op : Ops)
802 TmpVec.push_back(Elt: buildConstant(Res: EltTy, Val: Op));
803 return buildInstr(Opc: TargetOpcode::G_BUILD_VECTOR, DstOps: Res, SrcOps: TmpVec);
804}
805
806MachineInstrBuilder
807MachineIRBuilder::buildBuildVectorFConstant(const DstOp &Res,
808 ArrayRef<APFloat> Ops) {
809 SmallVector<SrcOp> TmpVec;
810 TmpVec.reserve(N: Ops.size());
811 LLT EltTy = Res.getLLTTy(MRI: *getMRI()).getElementType();
812 for (const auto &Op : Ops)
813 TmpVec.push_back(Elt: buildFConstant(Res: EltTy, Val: Op));
814 return buildInstr(Opc: TargetOpcode::G_BUILD_VECTOR, DstOps: Res, SrcOps: TmpVec);
815}
816
817MachineInstrBuilder MachineIRBuilder::buildSplatBuildVector(const DstOp &Res,
818 const SrcOp &Src) {
819 SmallVector<SrcOp, 8> TmpVec(Res.getLLTTy(MRI: *getMRI()).getNumElements(), Src);
820 return buildInstr(Opc: TargetOpcode::G_BUILD_VECTOR, DstOps: Res, SrcOps: TmpVec);
821}
822
823MachineInstrBuilder
824MachineIRBuilder::buildBuildVectorTrunc(const DstOp &Res,
825 ArrayRef<Register> Ops) {
826 // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
827 // we need some temporary storage for the DstOp objects. Here we use a
828 // sufficiently large SmallVector to not go through the heap.
829 SmallVector<SrcOp, 8> TmpVec(Ops);
830 if (TmpVec[0].getLLTTy(MRI: *getMRI()).getSizeInBits() ==
831 Res.getLLTTy(MRI: *getMRI()).getElementType().getSizeInBits())
832 return buildInstr(Opc: TargetOpcode::G_BUILD_VECTOR, DstOps: Res, SrcOps: TmpVec);
833 return buildInstr(Opc: TargetOpcode::G_BUILD_VECTOR_TRUNC, DstOps: Res, SrcOps: TmpVec);
834}
835
836MachineInstrBuilder MachineIRBuilder::buildShuffleSplat(const DstOp &Res,
837 const SrcOp &Src) {
838 LLT DstTy = Res.getLLTTy(MRI: *getMRI());
839 assert(Src.getLLTTy(*getMRI()) == DstTy.getElementType() &&
840 "Expected Src to match Dst elt ty");
841 auto UndefVec = buildUndef(Res: DstTy);
842 auto Zero = buildConstant(Res: LLT::integer(SizeInBits: 64), Val: 0);
843 auto InsElt = buildInsertVectorElement(Res: DstTy, Val: UndefVec, Elt: Src, Idx: Zero);
844 SmallVector<int, 16> ZeroMask(DstTy.getNumElements());
845 return buildShuffleVector(Res: DstTy, Src1: InsElt, Src2: UndefVec, Mask: ZeroMask);
846}
847
848MachineInstrBuilder MachineIRBuilder::buildSplatVector(const DstOp &Res,
849 const SrcOp &Src) {
850 assert(Src.getLLTTy(*getMRI()) == Res.getLLTTy(*getMRI()).getElementType() &&
851 "Expected Src to match Dst elt ty");
852 return buildInstr(Opc: TargetOpcode::G_SPLAT_VECTOR, DstOps: Res, SrcOps: Src);
853}
854
855MachineInstrBuilder MachineIRBuilder::buildShuffleVector(const DstOp &Res,
856 const SrcOp &Src1,
857 const SrcOp &Src2,
858 ArrayRef<int> Mask) {
859 LLT DstTy = Res.getLLTTy(MRI: *getMRI());
860 LLT Src1Ty = Src1.getLLTTy(MRI: *getMRI());
861 LLT Src2Ty = Src2.getLLTTy(MRI: *getMRI());
862 const LLT DstElemTy = DstTy.getScalarType();
863 const LLT ElemTy1 = Src1Ty.getScalarType();
864 const LLT ElemTy2 = Src2Ty.getScalarType();
865 assert(DstElemTy == ElemTy1 && DstElemTy == ElemTy2);
866 assert(Mask.size() > 1 && "Scalar G_SHUFFLE_VECTOR are not supported");
867 (void)DstElemTy;
868 (void)ElemTy1;
869 (void)ElemTy2;
870 ArrayRef<int> MaskAlloc = getMF().allocateShuffleMask(Mask);
871 return buildInstr(Opc: TargetOpcode::G_SHUFFLE_VECTOR, DstOps: {Res}, SrcOps: {Src1, Src2})
872 .addShuffleMask(Val: MaskAlloc);
873}
874
875MachineInstrBuilder
876MachineIRBuilder::buildConcatVectors(const DstOp &Res, ArrayRef<Register> Ops) {
877 // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
878 // we need some temporary storage for the DstOp objects. Here we use a
879 // sufficiently large SmallVector to not go through the heap.
880 SmallVector<SrcOp, 8> TmpVec(Ops);
881 return buildInstr(Opc: TargetOpcode::G_CONCAT_VECTORS, DstOps: Res, SrcOps: TmpVec);
882}
883
884MachineInstrBuilder MachineIRBuilder::buildInsert(const DstOp &Res,
885 const SrcOp &Src,
886 const SrcOp &Op,
887 unsigned Index) {
888 assert(Index + Op.getLLTTy(*getMRI()).getSizeInBits() <=
889 Res.getLLTTy(*getMRI()).getSizeInBits() &&
890 "insertion past the end of a register");
891
892 if (Res.getLLTTy(MRI: *getMRI()).getSizeInBits() ==
893 Op.getLLTTy(MRI: *getMRI()).getSizeInBits()) {
894 return buildCast(Dst: Res, Src: Op);
895 }
896
897 return buildInstr(Opc: TargetOpcode::G_INSERT, DstOps: Res, SrcOps: {Src, Op, uint64_t(Index)});
898}
899
900MachineInstrBuilder MachineIRBuilder::buildStepVector(const DstOp &Res,
901 unsigned Step) {
902 unsigned Bitwidth = Res.getLLTTy(MRI: *getMRI()).getElementType().getSizeInBits();
903 ConstantInt *CI = ConstantInt::get(Context&: getMF().getFunction().getContext(),
904 V: APInt(Bitwidth, Step));
905 auto StepVector = buildInstr(Opcode: TargetOpcode::G_STEP_VECTOR);
906 StepVector->setDebugLoc(DebugLoc());
907 Res.addDefToMIB(MRI&: *getMRI(), MIB&: StepVector);
908 StepVector.addCImm(Val: CI);
909 return StepVector;
910}
911
912MachineInstrBuilder MachineIRBuilder::buildVScale(const DstOp &Res,
913 unsigned MinElts) {
914
915 auto IntN = IntegerType::get(C&: getMF().getFunction().getContext(),
916 NumBits: Res.getLLTTy(MRI: *getMRI()).getScalarSizeInBits());
917 ConstantInt *CI = ConstantInt::get(Ty: IntN, V: MinElts);
918 return buildVScale(Res, MinElts: *CI);
919}
920
921MachineInstrBuilder MachineIRBuilder::buildVScale(const DstOp &Res,
922 const ConstantInt &MinElts) {
923 auto VScale = buildInstr(Opcode: TargetOpcode::G_VSCALE);
924 VScale->setDebugLoc(DebugLoc());
925 Res.addDefToMIB(MRI&: *getMRI(), MIB&: VScale);
926 VScale.addCImm(Val: &MinElts);
927 return VScale;
928}
929
930MachineInstrBuilder MachineIRBuilder::buildVScale(const DstOp &Res,
931 const APInt &MinElts) {
932 ConstantInt *CI =
933 ConstantInt::get(Context&: getMF().getFunction().getContext(), V: MinElts);
934 return buildVScale(Res, MinElts: *CI);
935}
936
937static unsigned getIntrinsicOpcode(bool HasSideEffects, bool IsConvergent) {
938 if (HasSideEffects && IsConvergent)
939 return TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS;
940 if (HasSideEffects)
941 return TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS;
942 if (IsConvergent)
943 return TargetOpcode::G_INTRINSIC_CONVERGENT;
944 return TargetOpcode::G_INTRINSIC;
945}
946
947MachineInstrBuilder
948MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
949 ArrayRef<Register> ResultRegs,
950 bool HasSideEffects, bool isConvergent) {
951 auto MIB = buildInstr(Opcode: getIntrinsicOpcode(HasSideEffects, IsConvergent: isConvergent));
952 for (Register ResultReg : ResultRegs)
953 MIB.addDef(RegNo: ResultReg);
954 MIB.addIntrinsicID(ID);
955 return MIB;
956}
957
958MachineInstrBuilder
959MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
960 ArrayRef<Register> ResultRegs) {
961 AttributeSet Attrs = Intrinsic::getFnAttributes(C&: getContext(), id: ID);
962 bool HasSideEffects = !Attrs.getMemoryEffects().doesNotAccessMemory();
963 bool isConvergent = Attrs.hasAttribute(Kind: Attribute::Convergent);
964 return buildIntrinsic(ID, ResultRegs, HasSideEffects, isConvergent);
965}
966
967MachineInstrBuilder MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
968 ArrayRef<DstOp> Results,
969 bool HasSideEffects,
970 bool isConvergent) {
971 auto MIB = buildInstr(Opcode: getIntrinsicOpcode(HasSideEffects, IsConvergent: isConvergent));
972 for (DstOp Result : Results)
973 Result.addDefToMIB(MRI&: *getMRI(), MIB);
974 MIB.addIntrinsicID(ID);
975 return MIB;
976}
977
978MachineInstrBuilder MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
979 ArrayRef<DstOp> Results) {
980 AttributeSet Attrs = Intrinsic::getFnAttributes(C&: getContext(), id: ID);
981 bool HasSideEffects = !Attrs.getMemoryEffects().doesNotAccessMemory();
982 bool isConvergent = Attrs.hasAttribute(Kind: Attribute::Convergent);
983 return buildIntrinsic(ID, Results, HasSideEffects, isConvergent);
984}
985
986MachineInstrBuilder
987MachineIRBuilder::buildTrunc(const DstOp &Res, const SrcOp &Op,
988 std::optional<unsigned> Flags) {
989 return buildInstr(Opc: TargetOpcode::G_TRUNC, DstOps: Res, SrcOps: Op, Flags);
990}
991
992MachineInstrBuilder
993MachineIRBuilder::buildFPTrunc(const DstOp &Res, const SrcOp &Op,
994 std::optional<unsigned> Flags) {
995 return buildInstr(Opc: TargetOpcode::G_FPTRUNC, DstOps: Res, SrcOps: Op, Flags);
996}
997
998MachineInstrBuilder MachineIRBuilder::buildICmp(CmpInst::Predicate Pred,
999 const DstOp &Res,
1000 const SrcOp &Op0,
1001 const SrcOp &Op1,
1002 std::optional<unsigned> Flags) {
1003 return buildInstr(Opc: TargetOpcode::G_ICMP, DstOps: Res, SrcOps: {Pred, Op0, Op1}, Flags);
1004}
1005
1006MachineInstrBuilder MachineIRBuilder::buildFCmp(CmpInst::Predicate Pred,
1007 const DstOp &Res,
1008 const SrcOp &Op0,
1009 const SrcOp &Op1,
1010 std::optional<unsigned> Flags) {
1011
1012 return buildInstr(Opc: TargetOpcode::G_FCMP, DstOps: Res, SrcOps: {Pred, Op0, Op1}, Flags);
1013}
1014
1015MachineInstrBuilder MachineIRBuilder::buildSCmp(const DstOp &Res,
1016 const SrcOp &Op0,
1017 const SrcOp &Op1) {
1018 return buildInstr(Opc: TargetOpcode::G_SCMP, DstOps: Res, SrcOps: {Op0, Op1});
1019}
1020
1021MachineInstrBuilder MachineIRBuilder::buildUCmp(const DstOp &Res,
1022 const SrcOp &Op0,
1023 const SrcOp &Op1) {
1024 return buildInstr(Opc: TargetOpcode::G_UCMP, DstOps: Res, SrcOps: {Op0, Op1});
1025}
1026
1027MachineInstrBuilder
1028MachineIRBuilder::buildSelect(const DstOp &Res, const SrcOp &Tst,
1029 const SrcOp &Op0, const SrcOp &Op1,
1030 std::optional<unsigned> Flags) {
1031
1032 return buildInstr(Opc: TargetOpcode::G_SELECT, DstOps: {Res}, SrcOps: {Tst, Op0, Op1}, Flags);
1033}
1034
1035MachineInstrBuilder MachineIRBuilder::buildInsertSubvector(const DstOp &Res,
1036 const SrcOp &Src0,
1037 const SrcOp &Src1,
1038 unsigned Idx) {
1039 return buildInstr(Opc: TargetOpcode::G_INSERT_SUBVECTOR, DstOps: Res,
1040 SrcOps: {Src0, Src1, uint64_t(Idx)});
1041}
1042
1043MachineInstrBuilder MachineIRBuilder::buildExtractSubvector(const DstOp &Res,
1044 const SrcOp &Src,
1045 unsigned Idx) {
1046 return buildInstr(Opc: TargetOpcode::G_EXTRACT_SUBVECTOR, DstOps: Res,
1047 SrcOps: {Src, uint64_t(Idx)});
1048}
1049
1050MachineInstrBuilder
1051MachineIRBuilder::buildInsertVectorElement(const DstOp &Res, const SrcOp &Val,
1052 const SrcOp &Elt, const SrcOp &Idx) {
1053 return buildInstr(Opc: TargetOpcode::G_INSERT_VECTOR_ELT, DstOps: Res, SrcOps: {Val, Elt, Idx});
1054}
1055
1056MachineInstrBuilder
1057MachineIRBuilder::buildExtractVectorElement(const DstOp &Res, const SrcOp &Val,
1058 const SrcOp &Idx) {
1059 return buildInstr(Opc: TargetOpcode::G_EXTRACT_VECTOR_ELT, DstOps: Res, SrcOps: {Val, Idx});
1060}
1061
1062MachineInstrBuilder MachineIRBuilder::buildAtomicCmpXchgWithSuccess(
1063 const DstOp &OldValRes, const DstOp &SuccessRes, const SrcOp &Addr,
1064 const SrcOp &CmpVal, const SrcOp &NewVal, MachineMemOperand &MMO) {
1065#ifndef NDEBUG
1066 LLT OldValResTy = OldValRes.getLLTTy(*getMRI());
1067 LLT SuccessResTy = SuccessRes.getLLTTy(*getMRI());
1068 LLT AddrTy = Addr.getLLTTy(*getMRI());
1069 LLT CmpValTy = CmpVal.getLLTTy(*getMRI());
1070 LLT NewValTy = NewVal.getLLTTy(*getMRI());
1071 assert(OldValResTy.isScalar() && "invalid operand type");
1072 assert(SuccessResTy.isScalar() && "invalid operand type");
1073 assert(AddrTy.isPointer() && "invalid operand type");
1074 assert(CmpValTy.isValid() && "invalid operand type");
1075 assert(NewValTy.isValid() && "invalid operand type");
1076 assert(OldValResTy == CmpValTy && "type mismatch");
1077 assert(OldValResTy == NewValTy && "type mismatch");
1078#endif
1079
1080 auto MIB = buildInstr(Opcode: TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS);
1081 OldValRes.addDefToMIB(MRI&: *getMRI(), MIB);
1082 SuccessRes.addDefToMIB(MRI&: *getMRI(), MIB);
1083 Addr.addSrcToMIB(MIB);
1084 CmpVal.addSrcToMIB(MIB);
1085 NewVal.addSrcToMIB(MIB);
1086 MIB.addMemOperand(MMO: &MMO);
1087 return MIB;
1088}
1089
1090MachineInstrBuilder
1091MachineIRBuilder::buildAtomicCmpXchg(const DstOp &OldValRes, const SrcOp &Addr,
1092 const SrcOp &CmpVal, const SrcOp &NewVal,
1093 MachineMemOperand &MMO) {
1094#ifndef NDEBUG
1095 LLT OldValResTy = OldValRes.getLLTTy(*getMRI());
1096 LLT AddrTy = Addr.getLLTTy(*getMRI());
1097 LLT CmpValTy = CmpVal.getLLTTy(*getMRI());
1098 LLT NewValTy = NewVal.getLLTTy(*getMRI());
1099 assert(OldValResTy.isScalar() && "invalid operand type");
1100 assert(AddrTy.isPointer() && "invalid operand type");
1101 assert(CmpValTy.isValid() && "invalid operand type");
1102 assert(NewValTy.isValid() && "invalid operand type");
1103 assert(OldValResTy == CmpValTy && "type mismatch");
1104 assert(OldValResTy == NewValTy && "type mismatch");
1105#endif
1106
1107 auto MIB = buildInstr(Opcode: TargetOpcode::G_ATOMIC_CMPXCHG);
1108 OldValRes.addDefToMIB(MRI&: *getMRI(), MIB);
1109 Addr.addSrcToMIB(MIB);
1110 CmpVal.addSrcToMIB(MIB);
1111 NewVal.addSrcToMIB(MIB);
1112 MIB.addMemOperand(MMO: &MMO);
1113 return MIB;
1114}
1115
1116MachineInstrBuilder MachineIRBuilder::buildAtomicRMW(
1117 unsigned Opcode, const DstOp &OldValRes,
1118 const SrcOp &Addr, const SrcOp &Val,
1119 MachineMemOperand &MMO) {
1120
1121#ifndef NDEBUG
1122 LLT OldValResTy = OldValRes.getLLTTy(*getMRI());
1123 LLT AddrTy = Addr.getLLTTy(*getMRI());
1124 LLT ValTy = Val.getLLTTy(*getMRI());
1125 assert(AddrTy.isPointer() && "invalid operand type");
1126 assert(ValTy.isValid() && "invalid operand type");
1127 assert(OldValResTy == ValTy && "type mismatch");
1128 assert(MMO.isAtomic() && "not atomic mem operand");
1129#endif
1130
1131 auto MIB = buildInstr(Opcode);
1132 OldValRes.addDefToMIB(MRI&: *getMRI(), MIB);
1133 Addr.addSrcToMIB(MIB);
1134 Val.addSrcToMIB(MIB);
1135 MIB.addMemOperand(MMO: &MMO);
1136 return MIB;
1137}
1138
1139MachineInstrBuilder
1140MachineIRBuilder::buildAtomicRMWXchg(Register OldValRes, Register Addr,
1141 Register Val, MachineMemOperand &MMO) {
1142 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_XCHG, OldValRes, Addr, Val,
1143 MMO);
1144}
1145MachineInstrBuilder
1146MachineIRBuilder::buildAtomicRMWAdd(Register OldValRes, Register Addr,
1147 Register Val, MachineMemOperand &MMO) {
1148 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_ADD, OldValRes, Addr, Val,
1149 MMO);
1150}
1151MachineInstrBuilder
1152MachineIRBuilder::buildAtomicRMWSub(Register OldValRes, Register Addr,
1153 Register Val, MachineMemOperand &MMO) {
1154 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_SUB, OldValRes, Addr, Val,
1155 MMO);
1156}
1157MachineInstrBuilder
1158MachineIRBuilder::buildAtomicRMWAnd(Register OldValRes, Register Addr,
1159 Register Val, MachineMemOperand &MMO) {
1160 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_AND, OldValRes, Addr, Val,
1161 MMO);
1162}
1163MachineInstrBuilder
1164MachineIRBuilder::buildAtomicRMWNand(Register OldValRes, Register Addr,
1165 Register Val, MachineMemOperand &MMO) {
1166 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_NAND, OldValRes, Addr, Val,
1167 MMO);
1168}
1169MachineInstrBuilder MachineIRBuilder::buildAtomicRMWOr(Register OldValRes,
1170 Register Addr,
1171 Register Val,
1172 MachineMemOperand &MMO) {
1173 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_OR, OldValRes, Addr, Val,
1174 MMO);
1175}
1176MachineInstrBuilder
1177MachineIRBuilder::buildAtomicRMWXor(Register OldValRes, Register Addr,
1178 Register Val, MachineMemOperand &MMO) {
1179 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_XOR, OldValRes, Addr, Val,
1180 MMO);
1181}
1182MachineInstrBuilder
1183MachineIRBuilder::buildAtomicRMWMax(Register OldValRes, Register Addr,
1184 Register Val, MachineMemOperand &MMO) {
1185 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_MAX, OldValRes, Addr, Val,
1186 MMO);
1187}
1188MachineInstrBuilder
1189MachineIRBuilder::buildAtomicRMWMin(Register OldValRes, Register Addr,
1190 Register Val, MachineMemOperand &MMO) {
1191 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_MIN, OldValRes, Addr, Val,
1192 MMO);
1193}
1194MachineInstrBuilder
1195MachineIRBuilder::buildAtomicRMWUmax(Register OldValRes, Register Addr,
1196 Register Val, MachineMemOperand &MMO) {
1197 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_UMAX, OldValRes, Addr, Val,
1198 MMO);
1199}
1200MachineInstrBuilder
1201MachineIRBuilder::buildAtomicRMWUmin(Register OldValRes, Register Addr,
1202 Register Val, MachineMemOperand &MMO) {
1203 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_UMIN, OldValRes, Addr, Val,
1204 MMO);
1205}
1206
1207MachineInstrBuilder
1208MachineIRBuilder::buildAtomicRMWFAdd(
1209 const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val,
1210 MachineMemOperand &MMO) {
1211 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_FADD, OldValRes, Addr, Val,
1212 MMO);
1213}
1214
1215MachineInstrBuilder
1216MachineIRBuilder::buildAtomicRMWFSub(const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val,
1217 MachineMemOperand &MMO) {
1218 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_FSUB, OldValRes, Addr, Val,
1219 MMO);
1220}
1221
1222MachineInstrBuilder
1223MachineIRBuilder::buildAtomicRMWFMax(const DstOp &OldValRes, const SrcOp &Addr,
1224 const SrcOp &Val, MachineMemOperand &MMO) {
1225 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_FMAX, OldValRes, Addr, Val,
1226 MMO);
1227}
1228
1229MachineInstrBuilder
1230MachineIRBuilder::buildAtomicRMWFMin(const DstOp &OldValRes, const SrcOp &Addr,
1231 const SrcOp &Val, MachineMemOperand &MMO) {
1232 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_FMIN, OldValRes, Addr, Val,
1233 MMO);
1234}
1235
1236MachineInstrBuilder
1237MachineIRBuilder::buildAtomicRMWFMaximum(const DstOp &OldValRes,
1238 const SrcOp &Addr, const SrcOp &Val,
1239 MachineMemOperand &MMO) {
1240 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_FMAXIMUM, OldValRes, Addr,
1241 Val, MMO);
1242}
1243
1244MachineInstrBuilder
1245MachineIRBuilder::buildAtomicRMWFMinimum(const DstOp &OldValRes,
1246 const SrcOp &Addr, const SrcOp &Val,
1247 MachineMemOperand &MMO) {
1248 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_FMINIMUM, OldValRes, Addr,
1249 Val, MMO);
1250}
1251
1252MachineInstrBuilder
1253MachineIRBuilder::buildFence(unsigned Ordering, unsigned Scope) {
1254 return buildInstr(Opcode: TargetOpcode::G_FENCE)
1255 .addImm(Val: Ordering)
1256 .addImm(Val: Scope);
1257}
1258
1259MachineInstrBuilder MachineIRBuilder::buildPrefetch(const SrcOp &Addr,
1260 unsigned RW,
1261 unsigned Locality,
1262 unsigned CacheType,
1263 MachineMemOperand &MMO) {
1264 auto MIB = buildInstr(Opcode: TargetOpcode::G_PREFETCH);
1265 Addr.addSrcToMIB(MIB);
1266 MIB.addImm(Val: RW).addImm(Val: Locality).addImm(Val: CacheType);
1267 MIB.addMemOperand(MMO: &MMO);
1268 return MIB;
1269}
1270
1271MachineInstrBuilder
1272MachineIRBuilder::buildBlockAddress(Register Res, const BlockAddress *BA) {
1273#ifndef NDEBUG
1274 assert(getMRI()->getType(Res).isPointer() && "invalid res type");
1275#endif
1276
1277 return buildInstr(Opcode: TargetOpcode::G_BLOCK_ADDR).addDef(RegNo: Res).addBlockAddress(BA);
1278}
1279
1280void MachineIRBuilder::validateTruncExt(const LLT DstTy, const LLT SrcTy,
1281 bool IsExtend) {
1282#ifndef NDEBUG
1283 if (DstTy.isVector()) {
1284 assert(SrcTy.isVector() && "mismatched cast between vector and non-vector");
1285 assert(SrcTy.getElementCount() == DstTy.getElementCount() &&
1286 "different number of elements in a trunc/ext");
1287 } else
1288 assert(DstTy.isScalar() && SrcTy.isScalar() && "invalid extend/trunc");
1289
1290 if (IsExtend)
1291 assert(TypeSize::isKnownGT(DstTy.getSizeInBits(), SrcTy.getSizeInBits()) &&
1292 "invalid narrowing extend");
1293 else
1294 assert(TypeSize::isKnownLT(DstTy.getSizeInBits(), SrcTy.getSizeInBits()) &&
1295 "invalid widening trunc");
1296#endif
1297}
1298
1299void MachineIRBuilder::validateSelectOp(const LLT ResTy, const LLT TstTy,
1300 const LLT Op0Ty, const LLT Op1Ty) {
1301#ifndef NDEBUG
1302 assert((ResTy.isScalar() || ResTy.isVector() || ResTy.isPointer()) &&
1303 "invalid operand type");
1304 assert((ResTy == Op0Ty && ResTy == Op1Ty) && "type mismatch");
1305 if (ResTy.isScalar() || ResTy.isPointer())
1306 assert(TstTy.isScalar() && "type mismatch");
1307 else
1308 assert((TstTy.isScalar() ||
1309 (TstTy.isVector() &&
1310 TstTy.getElementCount() == Op0Ty.getElementCount())) &&
1311 "type mismatch");
1312#endif
1313}
1314
1315MachineInstrBuilder
1316MachineIRBuilder::buildInstr(unsigned Opc, ArrayRef<DstOp> DstOps,
1317 ArrayRef<SrcOp> SrcOps,
1318 std::optional<unsigned> Flags) {
1319 switch (Opc) {
1320 default:
1321 break;
1322 case TargetOpcode::G_SELECT: {
1323 assert(DstOps.size() == 1 && "Invalid select");
1324 assert(SrcOps.size() == 3 && "Invalid select");
1325 validateSelectOp(
1326 ResTy: DstOps[0].getLLTTy(MRI: *getMRI()), TstTy: SrcOps[0].getLLTTy(MRI: *getMRI()),
1327 Op0Ty: SrcOps[1].getLLTTy(MRI: *getMRI()), Op1Ty: SrcOps[2].getLLTTy(MRI: *getMRI()));
1328 break;
1329 }
1330 case TargetOpcode::G_FNEG:
1331 case TargetOpcode::G_ABS:
1332 // All these are unary ops.
1333 assert(DstOps.size() == 1 && "Invalid Dst");
1334 assert(SrcOps.size() == 1 && "Invalid Srcs");
1335 validateUnaryOp(Res: DstOps[0].getLLTTy(MRI: *getMRI()),
1336 Op0: SrcOps[0].getLLTTy(MRI: *getMRI()));
1337 break;
1338 case TargetOpcode::G_ADD:
1339 case TargetOpcode::G_AND:
1340 case TargetOpcode::G_MUL:
1341 case TargetOpcode::G_OR:
1342 case TargetOpcode::G_SUB:
1343 case TargetOpcode::G_XOR:
1344 case TargetOpcode::G_UDIV:
1345 case TargetOpcode::G_SDIV:
1346 case TargetOpcode::G_UREM:
1347 case TargetOpcode::G_SREM:
1348 case TargetOpcode::G_SMIN:
1349 case TargetOpcode::G_SMAX:
1350 case TargetOpcode::G_UMIN:
1351 case TargetOpcode::G_UMAX:
1352 case TargetOpcode::G_UADDSAT:
1353 case TargetOpcode::G_SADDSAT:
1354 case TargetOpcode::G_USUBSAT:
1355 case TargetOpcode::G_SSUBSAT: {
1356 // All these are binary ops.
1357 assert(DstOps.size() == 1 && "Invalid Dst");
1358 assert(SrcOps.size() == 2 && "Invalid Srcs");
1359 validateBinaryOp(Res: DstOps[0].getLLTTy(MRI: *getMRI()),
1360 Op0: SrcOps[0].getLLTTy(MRI: *getMRI()),
1361 Op1: SrcOps[1].getLLTTy(MRI: *getMRI()));
1362 break;
1363 }
1364 case TargetOpcode::G_SHL:
1365 case TargetOpcode::G_ASHR:
1366 case TargetOpcode::G_LSHR:
1367 case TargetOpcode::G_USHLSAT:
1368 case TargetOpcode::G_SSHLSAT: {
1369 assert(DstOps.size() == 1 && "Invalid Dst");
1370 assert(SrcOps.size() == 2 && "Invalid Srcs");
1371 validateShiftOp(Res: DstOps[0].getLLTTy(MRI: *getMRI()),
1372 Op0: SrcOps[0].getLLTTy(MRI: *getMRI()),
1373 Op1: SrcOps[1].getLLTTy(MRI: *getMRI()));
1374 break;
1375 }
1376 case TargetOpcode::G_SEXT:
1377 case TargetOpcode::G_ZEXT:
1378 case TargetOpcode::G_ANYEXT:
1379 assert(DstOps.size() == 1 && "Invalid Dst");
1380 assert(SrcOps.size() == 1 && "Invalid Srcs");
1381 validateTruncExt(DstTy: DstOps[0].getLLTTy(MRI: *getMRI()),
1382 SrcTy: SrcOps[0].getLLTTy(MRI: *getMRI()), IsExtend: true);
1383 break;
1384 case TargetOpcode::G_TRUNC:
1385 case TargetOpcode::G_FPTRUNC: {
1386 assert(DstOps.size() == 1 && "Invalid Dst");
1387 assert(SrcOps.size() == 1 && "Invalid Srcs");
1388 validateTruncExt(DstTy: DstOps[0].getLLTTy(MRI: *getMRI()),
1389 SrcTy: SrcOps[0].getLLTTy(MRI: *getMRI()), IsExtend: false);
1390 break;
1391 }
1392 case TargetOpcode::G_BITCAST: {
1393 assert(DstOps.size() == 1 && "Invalid Dst");
1394 assert(SrcOps.size() == 1 && "Invalid Srcs");
1395 assert(DstOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1396 SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() && "invalid bitcast");
1397 break;
1398 }
1399 case TargetOpcode::COPY:
1400 assert(DstOps.size() == 1 && "Invalid Dst");
1401 // If the caller wants to add a subreg source it has to be done separately
1402 // so we may not have any SrcOps at this point yet.
1403 break;
1404 case TargetOpcode::G_FCMP:
1405 case TargetOpcode::G_ICMP: {
1406 assert(DstOps.size() == 1 && "Invalid Dst Operands");
1407 assert(SrcOps.size() == 3 && "Invalid Src Operands");
1408 // For F/ICMP, the first src operand is the predicate, followed by
1409 // the two comparands.
1410 assert(SrcOps[0].getSrcOpKind() == SrcOp::SrcType::Ty_Predicate &&
1411 "Expecting predicate");
1412 assert([&]() -> bool {
1413 CmpInst::Predicate Pred = SrcOps[0].getPredicate();
1414 return Opc == TargetOpcode::G_ICMP ? CmpInst::isIntPredicate(Pred)
1415 : CmpInst::isFPPredicate(Pred);
1416 }() && "Invalid predicate");
1417 assert(SrcOps[1].getLLTTy(*getMRI()) == SrcOps[2].getLLTTy(*getMRI()) &&
1418 "Type mismatch");
1419 assert([&]() -> bool {
1420 LLT Op0Ty = SrcOps[1].getLLTTy(*getMRI());
1421 LLT DstTy = DstOps[0].getLLTTy(*getMRI());
1422 if (Op0Ty.isScalar() || Op0Ty.isPointer())
1423 return DstTy.isScalar();
1424 else
1425 return DstTy.isVector() &&
1426 DstTy.getElementCount() == Op0Ty.getElementCount();
1427 }() && "Type Mismatch");
1428 break;
1429 }
1430 case TargetOpcode::G_UNMERGE_VALUES: {
1431 assert(!DstOps.empty() && "Invalid trivial sequence");
1432 assert(SrcOps.size() == 1 && "Invalid src for Unmerge");
1433 assert(llvm::all_of(DstOps,
1434 [&, this](const DstOp &Op) {
1435 return Op.getLLTTy(*getMRI()) ==
1436 DstOps[0].getLLTTy(*getMRI());
1437 }) &&
1438 "type mismatch in output list");
1439 assert((TypeSize::ScalarTy)DstOps.size() *
1440 DstOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1441 SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1442 "input operands do not cover output register");
1443 break;
1444 }
1445 case TargetOpcode::G_MERGE_VALUES: {
1446 assert(SrcOps.size() >= 2 && "invalid trivial sequence");
1447 assert(DstOps.size() == 1 && "Invalid Dst");
1448 assert(llvm::all_of(SrcOps,
1449 [&, this](const SrcOp &Op) {
1450 return Op.getLLTTy(*getMRI()) ==
1451 SrcOps[0].getLLTTy(*getMRI());
1452 }) &&
1453 "type mismatch in input list");
1454 assert((TypeSize::ScalarTy)SrcOps.size() *
1455 SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1456 DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1457 "input operands do not cover output register");
1458 assert(!DstOps[0].getLLTTy(*getMRI()).isVector() &&
1459 "vectors should be built with G_CONCAT_VECTOR or G_BUILD_VECTOR");
1460 break;
1461 }
1462 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
1463 assert(DstOps.size() == 1 && "Invalid Dst size");
1464 assert(SrcOps.size() == 2 && "Invalid Src size");
1465 assert(SrcOps[0].getLLTTy(*getMRI()).isVector() && "Invalid operand type");
1466 assert((DstOps[0].getLLTTy(*getMRI()).isScalar() ||
1467 DstOps[0].getLLTTy(*getMRI()).isPointer()) &&
1468 "Invalid operand type");
1469 assert(SrcOps[1].getLLTTy(*getMRI()).isScalar() && "Invalid operand type");
1470 assert(SrcOps[0].getLLTTy(*getMRI()).getElementType() ==
1471 DstOps[0].getLLTTy(*getMRI()) &&
1472 "Type mismatch");
1473 break;
1474 }
1475 case TargetOpcode::G_INSERT_VECTOR_ELT: {
1476 assert(DstOps.size() == 1 && "Invalid dst size");
1477 assert(SrcOps.size() == 3 && "Invalid src size");
1478 assert(DstOps[0].getLLTTy(*getMRI()).isVector() &&
1479 SrcOps[0].getLLTTy(*getMRI()).isVector() && "Invalid operand type");
1480 assert(DstOps[0].getLLTTy(*getMRI()).getElementType() ==
1481 SrcOps[1].getLLTTy(*getMRI()) &&
1482 "Type mismatch");
1483 assert(SrcOps[2].getLLTTy(*getMRI()).isScalar() && "Invalid index");
1484 assert(DstOps[0].getLLTTy(*getMRI()).getElementCount() ==
1485 SrcOps[0].getLLTTy(*getMRI()).getElementCount() &&
1486 "Type mismatch");
1487 break;
1488 }
1489 case TargetOpcode::G_INSERT_SUBVECTOR: {
1490 assert(DstOps.size() == 1 && "Invalid Dst");
1491 assert(SrcOps.size() == 3 && "Invalid Srcs");
1492 [[maybe_unused]] LLT DstTy = DstOps[0].getLLTTy(MRI: *getMRI());
1493 [[maybe_unused]] LLT BigVecTy = SrcOps[0].getLLTTy(MRI: *getMRI());
1494 [[maybe_unused]] LLT SubVecTy = SrcOps[1].getLLTTy(MRI: *getMRI());
1495 assert(DstTy == BigVecTy &&
1496 "Dest and insert subvector source types must match!");
1497 assert(DstTy.isVector() && SubVecTy.isVector() &&
1498 "Insert subvector VTs must be vectors!");
1499 assert(DstTy.getElementType() == SubVecTy.getElementType() &&
1500 "Insert subvector VTs must have the same element type!");
1501 assert((DstTy.isScalable() || !SubVecTy.isScalable()) &&
1502 "Cannot insert a scalable vector into a fixed length vector!");
1503 assert((DstTy.isScalable() != SubVecTy.isScalable() ||
1504 DstTy.getElementCount().getKnownMinValue() >=
1505 SubVecTy.getElementCount().getKnownMinValue()) &&
1506 "Insert subvector must be from smaller vector to larger vector!");
1507 assert(SrcOps[2].getSrcOpKind() == SrcOp::SrcType::Ty_Imm &&
1508 "Insert subvector index must be constant");
1509 assert((DstTy.isScalable() != SubVecTy.isScalable() ||
1510 (SubVecTy.getElementCount().getKnownMinValue() +
1511 (uint64_t)SrcOps[2].getImm()) <=
1512 DstTy.getElementCount().getKnownMinValue()) &&
1513 "Insert subvector overflow!");
1514 assert((uint64_t)SrcOps[2].getImm() %
1515 SubVecTy.getElementCount().getKnownMinValue() ==
1516 0 &&
1517 "Insert index is not a multiple of the subvector length");
1518 break;
1519 }
1520 case TargetOpcode::G_EXTRACT_SUBVECTOR: {
1521 assert(DstOps.size() == 1 && "Invalid Dst");
1522 assert(SrcOps.size() == 2 && "Invalid Srcs");
1523 [[maybe_unused]] LLT DstTy = DstOps[0].getLLTTy(MRI: *getMRI());
1524 [[maybe_unused]] LLT SrcVecTy = SrcOps[0].getLLTTy(MRI: *getMRI());
1525 assert(DstTy.isVector() && SrcVecTy.isVector() &&
1526 "Extract subvector VTs must be vectors!");
1527 assert(DstTy.getElementType() == SrcVecTy.getElementType() &&
1528 "Extract subvector VTs must have the same element type!");
1529 assert((!DstTy.isScalable() || SrcVecTy.isScalable()) &&
1530 "Cannot extract a scalable vector from a fixed length vector!");
1531 assert((DstTy.isScalable() != SrcVecTy.isScalable() ||
1532 DstTy.getElementCount().getKnownMinValue() <=
1533 SrcVecTy.getElementCount().getKnownMinValue()) &&
1534 "Extract subvector must be from larger vector to smaller vector!");
1535 assert(SrcOps[1].getSrcOpKind() == SrcOp::SrcType::Ty_Imm &&
1536 "Extract subvector index must be a constant");
1537 assert((DstTy.isScalable() != SrcVecTy.isScalable() ||
1538 (DstTy.getElementCount().getKnownMinValue() +
1539 (uint64_t)SrcOps[1].getImm()) <=
1540 SrcVecTy.getElementCount().getKnownMinValue()) &&
1541 "Extract subvector overflow!");
1542 assert((uint64_t)SrcOps[1].getImm() %
1543 DstTy.getElementCount().getKnownMinValue() ==
1544 0 &&
1545 "Extract index is not a multiple of the output vector length");
1546 break;
1547 }
1548 case TargetOpcode::G_BUILD_VECTOR: {
1549 assert((!SrcOps.empty() || SrcOps.size() < 2) &&
1550 "Must have at least 2 operands");
1551 assert(DstOps.size() == 1 && "Invalid DstOps");
1552 assert(DstOps[0].getLLTTy(*getMRI()).isVector() &&
1553 "Res type must be a vector");
1554 assert(llvm::all_of(SrcOps,
1555 [&, this](const SrcOp &Op) {
1556 return Op.getLLTTy(*getMRI()) ==
1557 SrcOps[0].getLLTTy(*getMRI());
1558 }) &&
1559 "type mismatch in input list");
1560 assert((TypeSize::ScalarTy)SrcOps.size() *
1561 SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1562 DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1563 "input scalars do not exactly cover the output vector register");
1564 break;
1565 }
1566 case TargetOpcode::G_BUILD_VECTOR_TRUNC: {
1567 assert((!SrcOps.empty() || SrcOps.size() < 2) &&
1568 "Must have at least 2 operands");
1569 assert(DstOps.size() == 1 && "Invalid DstOps");
1570 assert(DstOps[0].getLLTTy(*getMRI()).isVector() &&
1571 "Res type must be a vector");
1572 assert(llvm::all_of(SrcOps,
1573 [&, this](const SrcOp &Op) {
1574 return Op.getLLTTy(*getMRI()) ==
1575 SrcOps[0].getLLTTy(*getMRI());
1576 }) &&
1577 "type mismatch in input list");
1578 break;
1579 }
1580 case TargetOpcode::G_CONCAT_VECTORS: {
1581 assert(DstOps.size() == 1 && "Invalid DstOps");
1582 assert((!SrcOps.empty() || SrcOps.size() < 2) &&
1583 "Must have at least 2 operands");
1584 assert(llvm::all_of(SrcOps,
1585 [&, this](const SrcOp &Op) {
1586 return (Op.getLLTTy(*getMRI()).isVector() &&
1587 Op.getLLTTy(*getMRI()) ==
1588 SrcOps[0].getLLTTy(*getMRI()));
1589 }) &&
1590 "type mismatch in input list");
1591 assert((TypeSize::ScalarTy)SrcOps.size() *
1592 SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1593 DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1594 "input vectors do not exactly cover the output vector register");
1595 break;
1596 }
1597 case TargetOpcode::G_UADDE: {
1598 assert(DstOps.size() == 2 && "Invalid no of dst operands");
1599 assert(SrcOps.size() == 3 && "Invalid no of src operands");
1600 assert(DstOps[0].getLLTTy(*getMRI()).isScalar() && "Invalid operand");
1601 assert((DstOps[0].getLLTTy(*getMRI()) == SrcOps[0].getLLTTy(*getMRI())) &&
1602 (DstOps[0].getLLTTy(*getMRI()) == SrcOps[1].getLLTTy(*getMRI())) &&
1603 "Invalid operand");
1604 assert(DstOps[1].getLLTTy(*getMRI()).isScalar() && "Invalid operand");
1605 assert(DstOps[1].getLLTTy(*getMRI()) == SrcOps[2].getLLTTy(*getMRI()) &&
1606 "type mismatch");
1607 break;
1608 }
1609 }
1610
1611 auto MIB = buildInstr(Opcode: Opc);
1612 for (const DstOp &Op : DstOps)
1613 Op.addDefToMIB(MRI&: *getMRI(), MIB);
1614 for (const SrcOp &Op : SrcOps)
1615 Op.addSrcToMIB(MIB);
1616 if (Flags)
1617 MIB->setFlags(*Flags);
1618 return MIB;
1619}
1620