1//===-- llvm/CodeGen/GlobalISel/MachineIRBuilder.cpp - MIBuilder--*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the MachineIRBuidler class.
10//===----------------------------------------------------------------------===//
11#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
12#include "llvm/CodeGen/MachineFunction.h"
13#include "llvm/CodeGen/MachineInstr.h"
14#include "llvm/CodeGen/MachineInstrBuilder.h"
15#include "llvm/CodeGen/MachineRegisterInfo.h"
16#include "llvm/CodeGen/TargetInstrInfo.h"
17#include "llvm/CodeGen/TargetLowering.h"
18#include "llvm/CodeGen/TargetOpcodes.h"
19#include "llvm/CodeGen/TargetSubtargetInfo.h"
20#include "llvm/IR/DebugInfoMetadata.h"
21
22using namespace llvm;
23
24void MachineIRBuilder::setMF(MachineFunction &MF) {
25 State.MF = &MF;
26 State.MBB = nullptr;
27 State.MRI = &MF.getRegInfo();
28 State.TII = MF.getSubtarget().getInstrInfo();
29 State.DL = DebugLoc();
30 State.PCSections = nullptr;
31 State.MMRA = nullptr;
32 State.II = MachineBasicBlock::iterator();
33 State.Observer = nullptr;
34}
35
36//------------------------------------------------------------------------------
37// Build instruction variants.
38//------------------------------------------------------------------------------
39
40MachineInstrBuilder MachineIRBuilder::buildInstrNoInsert(unsigned Opcode) {
41 return BuildMI(
42 MF&: getMF(),
43 MIMD: {getDL(), getPCSections(), getMMRAMetadata(), getDeactivationSymbol()},
44 MCID: getTII().get(Opcode));
45}
46
47MachineInstrBuilder MachineIRBuilder::insertInstr(MachineInstrBuilder MIB) {
48 getMBB().insert(I: getInsertPt(), MI: MIB);
49 recordInsertion(InsertedInstr: MIB);
50 return MIB;
51}
52
53MachineInstrBuilder
54MachineIRBuilder::buildDirectDbgValue(Register Reg, const MDNode *Variable,
55 const MDNode *Expr) {
56 assert(isa<DILocalVariable>(Variable) && "not a variable");
57 assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
58 assert(
59 cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
60 "Expected inlined-at fields to agree");
61 return insertInstr(MIB: BuildMI(MF&: getMF(), DL: getDL(),
62 MCID: getTII().get(Opcode: TargetOpcode::DBG_VALUE),
63 /*IsIndirect*/ false, Reg, Variable, Expr));
64}
65
66MachineInstrBuilder
67MachineIRBuilder::buildIndirectDbgValue(Register Reg, const MDNode *Variable,
68 const MDNode *Expr) {
69 assert(isa<DILocalVariable>(Variable) && "not a variable");
70 assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
71 assert(
72 cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
73 "Expected inlined-at fields to agree");
74 return insertInstr(MIB: BuildMI(MF&: getMF(), DL: getDL(),
75 MCID: getTII().get(Opcode: TargetOpcode::DBG_VALUE),
76 /*IsIndirect*/ true, Reg, Variable, Expr));
77}
78
79MachineInstrBuilder MachineIRBuilder::buildFIDbgValue(int FI,
80 const MDNode *Variable,
81 const MDNode *Expr) {
82 assert(isa<DILocalVariable>(Variable) && "not a variable");
83 assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
84 assert(
85 cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
86 "Expected inlined-at fields to agree");
87 return insertInstr(MIB: buildInstrNoInsert(Opcode: TargetOpcode::DBG_VALUE)
88 .addFrameIndex(Idx: FI)
89 .addImm(Val: 0)
90 .addMetadata(MD: Variable)
91 .addMetadata(MD: Expr));
92}
93
94MachineInstrBuilder MachineIRBuilder::buildConstDbgValue(const Constant &C,
95 const MDNode *Variable,
96 const MDNode *Expr) {
97 assert(isa<DILocalVariable>(Variable) && "not a variable");
98 assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
99 assert(
100 cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
101 "Expected inlined-at fields to agree");
102 auto MIB = buildInstrNoInsert(Opcode: TargetOpcode::DBG_VALUE);
103
104 auto *NumericConstant = [&] () -> const Constant* {
105 if (const auto *CE = dyn_cast<ConstantExpr>(Val: &C))
106 if (CE->getOpcode() == Instruction::IntToPtr)
107 return CE->getOperand(i_nocapture: 0);
108 return &C;
109 }();
110
111 if (auto *CI = dyn_cast<ConstantInt>(Val: NumericConstant)) {
112 if (CI->getBitWidth() > 64)
113 MIB.addCImm(Val: CI);
114 else if (CI->getBitWidth() == 1)
115 MIB.addImm(Val: CI->getZExtValue());
116 else
117 MIB.addImm(Val: CI->getSExtValue());
118 } else if (auto *CFP = dyn_cast<ConstantFP>(Val: NumericConstant)) {
119 MIB.addFPImm(Val: CFP);
120 } else if (isa<ConstantPointerNull>(Val: NumericConstant)) {
121 MIB.addImm(Val: 0);
122 } else {
123 // Insert $noreg if we didn't find a usable constant and had to drop it.
124 MIB.addReg(RegNo: Register());
125 }
126
127 MIB.addImm(Val: 0).addMetadata(MD: Variable).addMetadata(MD: Expr);
128 return insertInstr(MIB);
129}
130
131MachineInstrBuilder MachineIRBuilder::buildDbgLabel(const MDNode *Label) {
132 assert(isa<DILabel>(Label) && "not a label");
133 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(State.DL) &&
134 "Expected inlined-at fields to agree");
135 auto MIB = buildInstr(Opcode: TargetOpcode::DBG_LABEL);
136
137 return MIB.addMetadata(MD: Label);
138}
139
140MachineInstrBuilder MachineIRBuilder::buildDynStackAlloc(const DstOp &Res,
141 const SrcOp &Size,
142 Align Alignment) {
143 assert(Res.getLLTTy(*getMRI()).isPointer() && "expected ptr dst type");
144 auto MIB = buildInstr(Opcode: TargetOpcode::G_DYN_STACKALLOC);
145 Res.addDefToMIB(MRI&: *getMRI(), MIB);
146 Size.addSrcToMIB(MIB);
147 MIB.addImm(Val: Alignment.value());
148 return MIB;
149}
150
151MachineInstrBuilder MachineIRBuilder::buildFrameIndex(const DstOp &Res,
152 int Idx) {
153 assert(Res.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
154 auto MIB = buildInstr(Opcode: TargetOpcode::G_FRAME_INDEX);
155 Res.addDefToMIB(MRI&: *getMRI(), MIB);
156 MIB.addFrameIndex(Idx);
157 return MIB;
158}
159
160MachineInstrBuilder MachineIRBuilder::buildGlobalValue(const DstOp &Res,
161 const GlobalValue *GV) {
162 assert(Res.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
163 assert(Res.getLLTTy(*getMRI()).getAddressSpace() ==
164 GV->getType()->getAddressSpace() &&
165 "address space mismatch");
166
167 auto MIB = buildInstr(Opcode: TargetOpcode::G_GLOBAL_VALUE);
168 Res.addDefToMIB(MRI&: *getMRI(), MIB);
169 MIB.addGlobalAddress(GV);
170 return MIB;
171}
172
173MachineInstrBuilder MachineIRBuilder::buildConstantPool(const DstOp &Res,
174 unsigned Idx) {
175 assert(Res.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
176 auto MIB = buildInstr(Opcode: TargetOpcode::G_CONSTANT_POOL);
177 Res.addDefToMIB(MRI&: *getMRI(), MIB);
178 MIB.addConstantPoolIndex(Idx);
179 return MIB;
180}
181
182MachineInstrBuilder MachineIRBuilder::buildJumpTable(const LLT PtrTy,
183 unsigned JTI) {
184 return buildInstr(Opc: TargetOpcode::G_JUMP_TABLE, DstOps: {PtrTy}, SrcOps: {})
185 .addJumpTableIndex(Idx: JTI);
186}
187
188void MachineIRBuilder::validateUnaryOp(const LLT Res, const LLT Op0) {
189 assert((Res.isScalar() || Res.isVector()) && "invalid operand type");
190 assert((Res == Op0) && "type mismatch");
191}
192
193void MachineIRBuilder::validateBinaryOp(const LLT Res, const LLT Op0,
194 const LLT Op1) {
195 assert((Res.isScalar() || Res.isVector()) && "invalid operand type");
196 assert((Res == Op0 && Res == Op1) && "type mismatch");
197}
198
199void MachineIRBuilder::validateShiftOp(const LLT Res, const LLT Op0,
200 const LLT Op1) {
201 assert((Res.isScalar() || Res.isVector()) && "invalid operand type");
202 assert((Res == Op0) && "type mismatch");
203}
204
205MachineInstrBuilder
206MachineIRBuilder::buildPtrAdd(const DstOp &Res, const SrcOp &Op0,
207 const SrcOp &Op1, std::optional<unsigned> Flags) {
208 assert(Res.getLLTTy(*getMRI()).isPointerOrPointerVector() &&
209 Res.getLLTTy(*getMRI()) == Op0.getLLTTy(*getMRI()) && "type mismatch");
210 assert(Op1.getLLTTy(*getMRI()).getScalarType().isScalar() && "invalid offset type");
211
212 return buildInstr(Opc: TargetOpcode::G_PTR_ADD, DstOps: {Res}, SrcOps: {Op0, Op1}, Flags);
213}
214
215MachineInstrBuilder MachineIRBuilder::buildObjectPtrOffset(const DstOp &Res,
216 const SrcOp &Op0,
217 const SrcOp &Op1) {
218 return buildPtrAdd(Res, Op0, Op1,
219 Flags: MachineInstr::MIFlag::NoUWrap |
220 MachineInstr::MIFlag::InBounds);
221}
222
223std::optional<MachineInstrBuilder>
224MachineIRBuilder::materializePtrAdd(Register &Res, Register Op0,
225 const LLT ValueTy, uint64_t Value,
226 std::optional<unsigned> Flags) {
227 assert(Res == 0 && "Res is a result argument");
228 assert(ValueTy.isScalar() && "invalid offset type");
229
230 if (Value == 0) {
231 Res = Op0;
232 return std::nullopt;
233 }
234
235 Res = getMRI()->createGenericVirtualRegister(Ty: getMRI()->getType(Reg: Op0));
236 auto Cst = buildConstant(Res: ValueTy, Val: Value);
237 return buildPtrAdd(Res, Op0, Op1: Cst.getReg(Idx: 0), Flags);
238}
239
240std::optional<MachineInstrBuilder> MachineIRBuilder::materializeObjectPtrOffset(
241 Register &Res, Register Op0, const LLT ValueTy, uint64_t Value) {
242 return materializePtrAdd(Res, Op0, ValueTy, Value,
243 Flags: MachineInstr::MIFlag::NoUWrap |
244 MachineInstr::MIFlag::InBounds);
245}
246
247MachineInstrBuilder MachineIRBuilder::buildMaskLowPtrBits(const DstOp &Res,
248 const SrcOp &Op0,
249 uint32_t NumBits) {
250 LLT PtrTy = Res.getLLTTy(MRI: *getMRI());
251 LLT MaskTy = LLT::scalar(SizeInBits: PtrTy.getSizeInBits());
252 Register MaskReg = getMRI()->createGenericVirtualRegister(Ty: MaskTy);
253 buildConstant(Res: MaskReg, Val: maskTrailingZeros<uint64_t>(N: NumBits));
254 return buildPtrMask(Res, Op0, Op1: MaskReg);
255}
256
257MachineInstrBuilder
258MachineIRBuilder::buildPadVectorWithUndefElements(const DstOp &Res,
259 const SrcOp &Op0) {
260 LLT ResTy = Res.getLLTTy(MRI: *getMRI());
261 LLT Op0Ty = Op0.getLLTTy(MRI: *getMRI());
262
263 assert(ResTy.isVector() && "Res non vector type");
264
265 SmallVector<Register, 8> Regs;
266 if (Op0Ty.isVector()) {
267 assert((ResTy.getElementType() == Op0Ty.getElementType()) &&
268 "Different vector element types");
269 assert((ResTy.getNumElements() > Op0Ty.getNumElements()) &&
270 "Op0 has more elements");
271 auto Unmerge = buildUnmerge(Res: Op0Ty.getElementType(), Op: Op0);
272
273 for (auto Op : Unmerge.getInstr()->defs())
274 Regs.push_back(Elt: Op.getReg());
275 } else {
276 assert((ResTy.getSizeInBits() > Op0Ty.getSizeInBits()) &&
277 "Op0 has more size");
278 Regs.push_back(Elt: Op0.getReg());
279 }
280 Register Undef =
281 buildUndef(Res: Op0Ty.isVector() ? Op0Ty.getElementType() : Op0Ty).getReg(Idx: 0);
282 unsigned NumberOfPadElts = ResTy.getNumElements() - Regs.size();
283 for (unsigned i = 0; i < NumberOfPadElts; ++i)
284 Regs.push_back(Elt: Undef);
285 return buildMergeLikeInstr(Res, Ops: Regs);
286}
287
288MachineInstrBuilder
289MachineIRBuilder::buildDeleteTrailingVectorElements(const DstOp &Res,
290 const SrcOp &Op0) {
291 LLT ResTy = Res.getLLTTy(MRI: *getMRI());
292 LLT Op0Ty = Op0.getLLTTy(MRI: *getMRI());
293
294 assert(Op0Ty.isVector() && "Non vector type");
295 assert(((ResTy.isScalar() && (ResTy == Op0Ty.getElementType())) ||
296 (ResTy.isVector() &&
297 (ResTy.getElementType() == Op0Ty.getElementType()))) &&
298 "Different vector element types");
299 assert(
300 (ResTy.isScalar() || (ResTy.getNumElements() < Op0Ty.getNumElements())) &&
301 "Op0 has fewer elements");
302
303 auto Unmerge = buildUnmerge(Res: Op0Ty.getElementType(), Op: Op0);
304 if (ResTy.isScalar())
305 return buildCopy(Res, Op: Unmerge.getReg(Idx: 0));
306 SmallVector<Register, 8> Regs;
307 for (unsigned i = 0; i < ResTy.getNumElements(); ++i)
308 Regs.push_back(Elt: Unmerge.getReg(Idx: i));
309 return buildMergeLikeInstr(Res, Ops: Regs);
310}
311
312MachineInstrBuilder MachineIRBuilder::buildBr(MachineBasicBlock &Dest) {
313 return buildInstr(Opcode: TargetOpcode::G_BR).addMBB(MBB: &Dest);
314}
315
316MachineInstrBuilder MachineIRBuilder::buildBrIndirect(Register Tgt) {
317 assert(getMRI()->getType(Tgt).isPointer() && "invalid branch destination");
318 return buildInstr(Opcode: TargetOpcode::G_BRINDIRECT).addUse(RegNo: Tgt);
319}
320
321MachineInstrBuilder MachineIRBuilder::buildBrJT(Register TablePtr,
322 unsigned JTI,
323 Register IndexReg) {
324 assert(getMRI()->getType(TablePtr).isPointer() &&
325 "Table reg must be a pointer");
326 return buildInstr(Opcode: TargetOpcode::G_BRJT)
327 .addUse(RegNo: TablePtr)
328 .addJumpTableIndex(Idx: JTI)
329 .addUse(RegNo: IndexReg);
330}
331
332MachineInstrBuilder MachineIRBuilder::buildCopy(const DstOp &Res,
333 const SrcOp &Op) {
334 return buildInstr(Opc: TargetOpcode::COPY, DstOps: Res, SrcOps: Op);
335}
336
337MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
338 const ConstantInt &Val) {
339 assert(!isa<VectorType>(Val.getType()) && "Unexpected vector constant!");
340 LLT Ty = Res.getLLTTy(MRI: *getMRI());
341 LLT EltTy = Ty.getScalarType();
342 assert(EltTy.getScalarSizeInBits() == Val.getBitWidth() &&
343 "creating constant with the wrong size");
344
345 assert(!Ty.isScalableVector() &&
346 "unexpected scalable vector in buildConstant");
347
348 if (Ty.isFixedVector()) {
349 auto Const = buildInstr(Opcode: TargetOpcode::G_CONSTANT)
350 .addDef(RegNo: getMRI()->createGenericVirtualRegister(Ty: EltTy))
351 .addCImm(Val: &Val);
352 return buildSplatBuildVector(Res, Src: Const);
353 }
354
355 auto Const = buildInstr(Opcode: TargetOpcode::G_CONSTANT);
356 Const->setDebugLoc(DebugLoc());
357 Res.addDefToMIB(MRI&: *getMRI(), MIB&: Const);
358 Const.addCImm(Val: &Val);
359 return Const;
360}
361
362MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
363 int64_t Val) {
364 auto IntN = IntegerType::get(C&: getMF().getFunction().getContext(),
365 NumBits: Res.getLLTTy(MRI: *getMRI()).getScalarSizeInBits());
366 // TODO: Avoid implicit trunc?
367 // See https://github.com/llvm/llvm-project/issues/112510.
368 ConstantInt *CI = ConstantInt::getSigned(Ty: IntN, V: Val, /*implicitTrunc=*/ImplicitTrunc: true);
369 return buildConstant(Res, Val: *CI);
370}
371
372MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
373 const ConstantFP &Val) {
374 assert(!isa<VectorType>(Val.getType()) && "Unexpected vector constant!");
375 LLT Ty = Res.getLLTTy(MRI: *getMRI());
376 LLT EltTy = Ty.getScalarType();
377
378 assert(APFloat::getSizeInBits(Val.getValueAPF().getSemantics())
379 == EltTy.getSizeInBits() &&
380 "creating fconstant with the wrong size");
381
382 assert(!Ty.isPointer() && "invalid operand type");
383
384 assert(!Ty.isScalableVector() &&
385 "unexpected scalable vector in buildFConstant");
386
387 if (Ty.isFixedVector()) {
388 auto Const = buildInstr(Opcode: TargetOpcode::G_FCONSTANT)
389 .addDef(RegNo: getMRI()->createGenericVirtualRegister(Ty: EltTy))
390 .addFPImm(Val: &Val);
391
392 return buildSplatBuildVector(Res, Src: Const);
393 }
394
395 auto Const = buildInstr(Opcode: TargetOpcode::G_FCONSTANT);
396 Const->setDebugLoc(DebugLoc());
397 Res.addDefToMIB(MRI&: *getMRI(), MIB&: Const);
398 Const.addFPImm(Val: &Val);
399 return Const;
400}
401
402MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
403 const APInt &Val) {
404 ConstantInt *CI = ConstantInt::get(Context&: getMF().getFunction().getContext(), V: Val);
405 return buildConstant(Res, Val: *CI);
406}
407
408MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
409 double Val) {
410 LLT DstTy = Res.getLLTTy(MRI: *getMRI());
411 auto &Ctx = getMF().getFunction().getContext();
412 auto *CFP =
413 ConstantFP::get(Context&: Ctx, V: getAPFloatFromSize(Val, Size: DstTy.getScalarSizeInBits()));
414 return buildFConstant(Res, Val: *CFP);
415}
416
417MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
418 const APFloat &Val) {
419 auto &Ctx = getMF().getFunction().getContext();
420 auto *CFP = ConstantFP::get(Context&: Ctx, V: Val);
421 return buildFConstant(Res, Val: *CFP);
422}
423
424MachineInstrBuilder
425MachineIRBuilder::buildConstantPtrAuth(const DstOp &Res,
426 const ConstantPtrAuth *CPA,
427 Register Addr, Register AddrDisc) {
428 auto MIB = buildInstr(Opcode: TargetOpcode::G_PTRAUTH_GLOBAL_VALUE);
429 Res.addDefToMIB(MRI&: *getMRI(), MIB);
430 MIB.addUse(RegNo: Addr);
431 MIB.addImm(Val: CPA->getKey()->getZExtValue());
432 MIB.addUse(RegNo: AddrDisc);
433 MIB.addImm(Val: CPA->getDiscriminator()->getZExtValue());
434 return MIB;
435}
436
437MachineInstrBuilder MachineIRBuilder::buildBrCond(const SrcOp &Tst,
438 MachineBasicBlock &Dest) {
439 assert(Tst.getLLTTy(*getMRI()).isScalar() && "invalid operand type");
440
441 auto MIB = buildInstr(Opcode: TargetOpcode::G_BRCOND);
442 Tst.addSrcToMIB(MIB);
443 MIB.addMBB(MBB: &Dest);
444 return MIB;
445}
446
447MachineInstrBuilder
448MachineIRBuilder::buildLoad(const DstOp &Dst, const SrcOp &Addr,
449 MachinePointerInfo PtrInfo, Align Alignment,
450 MachineMemOperand::Flags MMOFlags,
451 const AAMDNodes &AAInfo) {
452 MMOFlags |= MachineMemOperand::MOLoad;
453 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
454
455 LLT Ty = Dst.getLLTTy(MRI: *getMRI());
456 MachineMemOperand *MMO =
457 getMF().getMachineMemOperand(PtrInfo, f: MMOFlags, MemTy: Ty, base_alignment: Alignment, AAInfo);
458 return buildLoad(Res: Dst, Addr, MMO&: *MMO);
459}
460
461MachineInstrBuilder MachineIRBuilder::buildLoadInstr(unsigned Opcode,
462 const DstOp &Res,
463 const SrcOp &Addr,
464 MachineMemOperand &MMO) {
465 assert(Res.getLLTTy(*getMRI()).isValid() && "invalid operand type");
466 assert(Addr.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
467
468 auto MIB = buildInstr(Opcode);
469 Res.addDefToMIB(MRI&: *getMRI(), MIB);
470 Addr.addSrcToMIB(MIB);
471 MIB.addMemOperand(MMO: &MMO);
472 return MIB;
473}
474
475MachineInstrBuilder MachineIRBuilder::buildLoadFromOffset(
476 const DstOp &Dst, const SrcOp &BasePtr,
477 MachineMemOperand &BaseMMO, int64_t Offset) {
478 LLT LoadTy = Dst.getLLTTy(MRI: *getMRI());
479 MachineMemOperand *OffsetMMO =
480 getMF().getMachineMemOperand(MMO: &BaseMMO, Offset, Ty: LoadTy);
481
482 if (Offset == 0) // This may be a size or type changing load.
483 return buildLoad(Res: Dst, Addr: BasePtr, MMO&: *OffsetMMO);
484
485 LLT PtrTy = BasePtr.getLLTTy(MRI: *getMRI());
486 LLT OffsetTy = LLT::scalar(SizeInBits: PtrTy.getSizeInBits());
487 auto ConstOffset = buildConstant(Res: OffsetTy, Val: Offset);
488 auto Ptr = buildPtrAdd(Res: PtrTy, Op0: BasePtr, Op1: ConstOffset);
489 return buildLoad(Res: Dst, Addr: Ptr, MMO&: *OffsetMMO);
490}
491
492MachineInstrBuilder MachineIRBuilder::buildStore(const SrcOp &Val,
493 const SrcOp &Addr,
494 MachineMemOperand &MMO) {
495 assert(Val.getLLTTy(*getMRI()).isValid() && "invalid operand type");
496 assert(Addr.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
497
498 auto MIB = buildInstr(Opcode: TargetOpcode::G_STORE);
499 Val.addSrcToMIB(MIB);
500 Addr.addSrcToMIB(MIB);
501 MIB.addMemOperand(MMO: &MMO);
502 return MIB;
503}
504
505MachineInstrBuilder MachineIRBuilder::buildStoreInstr(unsigned Opcode,
506 const SrcOp &Val,
507 const SrcOp &Addr,
508 MachineMemOperand &MMO) {
509 assert(Val.getLLTTy(*getMRI()).isValid() && "invalid operand type");
510 assert(Addr.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
511
512 auto MIB = buildInstr(Opcode);
513 Val.addSrcToMIB(MIB);
514 Addr.addSrcToMIB(MIB);
515 MIB.addMemOperand(MMO: &MMO);
516 return MIB;
517}
518
519MachineInstrBuilder
520MachineIRBuilder::buildStore(const SrcOp &Val, const SrcOp &Addr,
521 MachinePointerInfo PtrInfo, Align Alignment,
522 MachineMemOperand::Flags MMOFlags,
523 const AAMDNodes &AAInfo) {
524 MMOFlags |= MachineMemOperand::MOStore;
525 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
526
527 LLT Ty = Val.getLLTTy(MRI: *getMRI());
528 MachineMemOperand *MMO =
529 getMF().getMachineMemOperand(PtrInfo, f: MMOFlags, MemTy: Ty, base_alignment: Alignment, AAInfo);
530 return buildStore(Val, Addr, MMO&: *MMO);
531}
532
533MachineInstrBuilder MachineIRBuilder::buildAnyExt(const DstOp &Res,
534 const SrcOp &Op) {
535 return buildInstr(Opc: TargetOpcode::G_ANYEXT, DstOps: Res, SrcOps: Op);
536}
537
538MachineInstrBuilder MachineIRBuilder::buildSExt(const DstOp &Res,
539 const SrcOp &Op) {
540 return buildInstr(Opc: TargetOpcode::G_SEXT, DstOps: Res, SrcOps: Op);
541}
542
543MachineInstrBuilder MachineIRBuilder::buildZExt(const DstOp &Res,
544 const SrcOp &Op,
545 std::optional<unsigned> Flags) {
546 return buildInstr(Opc: TargetOpcode::G_ZEXT, DstOps: Res, SrcOps: Op, Flags);
547}
548
549unsigned MachineIRBuilder::getBoolExtOp(bool IsVec, bool IsFP) const {
550 const auto *TLI = getMF().getSubtarget().getTargetLowering();
551 switch (TLI->getBooleanContents(isVec: IsVec, isFloat: IsFP)) {
552 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
553 return TargetOpcode::G_SEXT;
554 case TargetLoweringBase::ZeroOrOneBooleanContent:
555 return TargetOpcode::G_ZEXT;
556 default:
557 return TargetOpcode::G_ANYEXT;
558 }
559}
560
561MachineInstrBuilder MachineIRBuilder::buildBoolExt(const DstOp &Res,
562 const SrcOp &Op,
563 bool IsFP) {
564 unsigned ExtOp = getBoolExtOp(IsVec: getMRI()->getType(Reg: Op.getReg()).isVector(), IsFP);
565 return buildInstr(Opc: ExtOp, DstOps: Res, SrcOps: Op);
566}
567
568MachineInstrBuilder MachineIRBuilder::buildBoolExtInReg(const DstOp &Res,
569 const SrcOp &Op,
570 bool IsVector,
571 bool IsFP) {
572 const auto *TLI = getMF().getSubtarget().getTargetLowering();
573 switch (TLI->getBooleanContents(isVec: IsVector, isFloat: IsFP)) {
574 case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
575 return buildSExtInReg(Res, Op, ImmOp: 1);
576 case TargetLoweringBase::ZeroOrOneBooleanContent:
577 return buildZExtInReg(Res, Op, ImmOp: 1);
578 case TargetLoweringBase::UndefinedBooleanContent:
579 return buildCopy(Res, Op);
580 }
581
582 llvm_unreachable("unexpected BooleanContent");
583}
584
585MachineInstrBuilder MachineIRBuilder::buildExtOrTrunc(unsigned ExtOpc,
586 const DstOp &Res,
587 const SrcOp &Op) {
588 assert((TargetOpcode::G_ANYEXT == ExtOpc || TargetOpcode::G_ZEXT == ExtOpc ||
589 TargetOpcode::G_SEXT == ExtOpc) &&
590 "Expecting Extending Opc");
591 assert(Res.getLLTTy(*getMRI()).isScalar() ||
592 Res.getLLTTy(*getMRI()).isVector());
593 assert(Res.getLLTTy(*getMRI()).isScalar() ==
594 Op.getLLTTy(*getMRI()).isScalar());
595
596 unsigned Opcode = TargetOpcode::COPY;
597 if (Res.getLLTTy(MRI: *getMRI()).getSizeInBits() >
598 Op.getLLTTy(MRI: *getMRI()).getSizeInBits())
599 Opcode = ExtOpc;
600 else if (Res.getLLTTy(MRI: *getMRI()).getSizeInBits() <
601 Op.getLLTTy(MRI: *getMRI()).getSizeInBits())
602 Opcode = TargetOpcode::G_TRUNC;
603 else
604 assert(Res.getLLTTy(*getMRI()).getSizeInBits() ==
605 Op.getLLTTy(*getMRI()).getSizeInBits());
606
607 return buildInstr(Opc: Opcode, DstOps: Res, SrcOps: Op);
608}
609
610MachineInstrBuilder MachineIRBuilder::buildSExtOrTrunc(const DstOp &Res,
611 const SrcOp &Op) {
612 return buildExtOrTrunc(ExtOpc: TargetOpcode::G_SEXT, Res, Op);
613}
614
615MachineInstrBuilder MachineIRBuilder::buildZExtOrTrunc(const DstOp &Res,
616 const SrcOp &Op) {
617 return buildExtOrTrunc(ExtOpc: TargetOpcode::G_ZEXT, Res, Op);
618}
619
620MachineInstrBuilder MachineIRBuilder::buildAnyExtOrTrunc(const DstOp &Res,
621 const SrcOp &Op) {
622 return buildExtOrTrunc(ExtOpc: TargetOpcode::G_ANYEXT, Res, Op);
623}
624
625MachineInstrBuilder MachineIRBuilder::buildZExtInReg(const DstOp &Res,
626 const SrcOp &Op,
627 int64_t ImmOp) {
628 LLT ResTy = Res.getLLTTy(MRI: *getMRI());
629 auto Mask = buildConstant(
630 Res: ResTy, Val: APInt::getLowBitsSet(numBits: ResTy.getScalarSizeInBits(), loBitsSet: ImmOp));
631 return buildAnd(Dst: Res, Src0: Op, Src1: Mask);
632}
633
634MachineInstrBuilder MachineIRBuilder::buildCast(const DstOp &Dst,
635 const SrcOp &Src) {
636 LLT SrcTy = Src.getLLTTy(MRI: *getMRI());
637 LLT DstTy = Dst.getLLTTy(MRI: *getMRI());
638 if (SrcTy == DstTy)
639 return buildCopy(Res: Dst, Op: Src);
640
641 unsigned Opcode;
642 if (SrcTy.isPointerOrPointerVector())
643 Opcode = TargetOpcode::G_PTRTOINT;
644 else if (DstTy.isPointerOrPointerVector())
645 Opcode = TargetOpcode::G_INTTOPTR;
646 else {
647 assert(!SrcTy.isPointerOrPointerVector() &&
648 !DstTy.isPointerOrPointerVector() && "no G_ADDRCAST yet");
649 Opcode = TargetOpcode::G_BITCAST;
650 }
651
652 return buildInstr(Opc: Opcode, DstOps: Dst, SrcOps: Src);
653}
654
655MachineInstrBuilder MachineIRBuilder::buildExtract(const DstOp &Dst,
656 const SrcOp &Src,
657 uint64_t Index) {
658 LLT SrcTy = Src.getLLTTy(MRI: *getMRI());
659 LLT DstTy = Dst.getLLTTy(MRI: *getMRI());
660
661#ifndef NDEBUG
662 assert(SrcTy.isValid() && "invalid operand type");
663 assert(DstTy.isValid() && "invalid operand type");
664 assert(Index + DstTy.getSizeInBits() <= SrcTy.getSizeInBits() &&
665 "extracting off end of register");
666#endif
667
668 if (DstTy.getSizeInBits() == SrcTy.getSizeInBits()) {
669 assert(Index == 0 && "insertion past the end of a register");
670 return buildCast(Dst, Src);
671 }
672
673 auto Extract = buildInstr(Opcode: TargetOpcode::G_EXTRACT);
674 Dst.addDefToMIB(MRI&: *getMRI(), MIB&: Extract);
675 Src.addSrcToMIB(MIB&: Extract);
676 Extract.addImm(Val: Index);
677 return Extract;
678}
679
680MachineInstrBuilder MachineIRBuilder::buildUndef(const DstOp &Res) {
681 return buildInstr(Opc: TargetOpcode::G_IMPLICIT_DEF, DstOps: {Res}, SrcOps: {});
682}
683
684MachineInstrBuilder MachineIRBuilder::buildMergeValues(const DstOp &Res,
685 ArrayRef<Register> Ops) {
686 // Unfortunately to convert from ArrayRef<LLT> to ArrayRef<SrcOp>,
687 // we need some temporary storage for the DstOp objects. Here we use a
688 // sufficiently large SmallVector to not go through the heap.
689 SmallVector<SrcOp, 8> TmpVec(Ops);
690 assert(TmpVec.size() > 1);
691 return buildInstr(Opc: TargetOpcode::G_MERGE_VALUES, DstOps: Res, SrcOps: TmpVec);
692}
693
694MachineInstrBuilder
695MachineIRBuilder::buildMergeLikeInstr(const DstOp &Res,
696 ArrayRef<Register> Ops) {
697 // Unfortunately to convert from ArrayRef<LLT> to ArrayRef<SrcOp>,
698 // we need some temporary storage for the DstOp objects. Here we use a
699 // sufficiently large SmallVector to not go through the heap.
700 SmallVector<SrcOp, 8> TmpVec(Ops);
701 assert(TmpVec.size() > 1);
702 return buildInstr(Opc: getOpcodeForMerge(DstOp: Res, SrcOps: TmpVec), DstOps: Res, SrcOps: TmpVec);
703}
704
705MachineInstrBuilder
706MachineIRBuilder::buildMergeLikeInstr(const DstOp &Res,
707 std::initializer_list<SrcOp> Ops) {
708 assert(Ops.size() > 1);
709 return buildInstr(Opc: getOpcodeForMerge(DstOp: Res, SrcOps: Ops), DstOps: Res, SrcOps: Ops);
710}
711
712unsigned MachineIRBuilder::getOpcodeForMerge(const DstOp &DstOp,
713 ArrayRef<SrcOp> SrcOps) const {
714 if (DstOp.getLLTTy(MRI: *getMRI()).isVector()) {
715 if (SrcOps[0].getLLTTy(MRI: *getMRI()).isVector())
716 return TargetOpcode::G_CONCAT_VECTORS;
717 return TargetOpcode::G_BUILD_VECTOR;
718 }
719
720 return TargetOpcode::G_MERGE_VALUES;
721}
722
723MachineInstrBuilder MachineIRBuilder::buildUnmerge(ArrayRef<LLT> Res,
724 const SrcOp &Op) {
725 // Unfortunately to convert from ArrayRef<LLT> to ArrayRef<DstOp>,
726 // we need some temporary storage for the DstOp objects. Here we use a
727 // sufficiently large SmallVector to not go through the heap.
728 SmallVector<DstOp, 8> TmpVec(Res);
729 assert(TmpVec.size() > 1);
730 return buildInstr(Opc: TargetOpcode::G_UNMERGE_VALUES, DstOps: TmpVec, SrcOps: Op);
731}
732
733MachineInstrBuilder MachineIRBuilder::buildUnmerge(LLT Res,
734 const SrcOp &Op) {
735 unsigned NumReg = Op.getLLTTy(MRI: *getMRI()).getSizeInBits() / Res.getSizeInBits();
736 SmallVector<DstOp, 8> TmpVec(NumReg, Res);
737 return buildInstr(Opc: TargetOpcode::G_UNMERGE_VALUES, DstOps: TmpVec, SrcOps: Op);
738}
739
740MachineInstrBuilder
741MachineIRBuilder::buildUnmerge(MachineRegisterInfo::VRegAttrs Attrs,
742 const SrcOp &Op) {
743 LLT OpTy = Op.getLLTTy(MRI: *getMRI());
744 unsigned NumRegs = OpTy.getSizeInBits() / Attrs.Ty.getSizeInBits();
745 SmallVector<DstOp, 8> TmpVec(NumRegs, Attrs);
746 return buildInstr(Opc: TargetOpcode::G_UNMERGE_VALUES, DstOps: TmpVec, SrcOps: Op);
747}
748
749MachineInstrBuilder MachineIRBuilder::buildUnmerge(ArrayRef<Register> Res,
750 const SrcOp &Op) {
751 // Unfortunately to convert from ArrayRef<Register> to ArrayRef<DstOp>,
752 // we need some temporary storage for the DstOp objects. Here we use a
753 // sufficiently large SmallVector to not go through the heap.
754 SmallVector<DstOp, 8> TmpVec(Res);
755 assert(TmpVec.size() > 1);
756 return buildInstr(Opc: TargetOpcode::G_UNMERGE_VALUES, DstOps: TmpVec, SrcOps: Op);
757}
758
759MachineInstrBuilder MachineIRBuilder::buildBuildVector(const DstOp &Res,
760 ArrayRef<Register> Ops) {
761 // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
762 // we need some temporary storage for the DstOp objects. Here we use a
763 // sufficiently large SmallVector to not go through the heap.
764 SmallVector<SrcOp, 8> TmpVec(Ops);
765 return buildInstr(Opc: TargetOpcode::G_BUILD_VECTOR, DstOps: Res, SrcOps: TmpVec);
766}
767
768MachineInstrBuilder
769MachineIRBuilder::buildBuildVectorConstant(const DstOp &Res,
770 ArrayRef<APInt> Ops) {
771 SmallVector<SrcOp> TmpVec;
772 TmpVec.reserve(N: Ops.size());
773 LLT EltTy = Res.getLLTTy(MRI: *getMRI()).getElementType();
774 for (const auto &Op : Ops)
775 TmpVec.push_back(Elt: buildConstant(Res: EltTy, Val: Op));
776 return buildInstr(Opc: TargetOpcode::G_BUILD_VECTOR, DstOps: Res, SrcOps: TmpVec);
777}
778
779MachineInstrBuilder MachineIRBuilder::buildSplatBuildVector(const DstOp &Res,
780 const SrcOp &Src) {
781 SmallVector<SrcOp, 8> TmpVec(Res.getLLTTy(MRI: *getMRI()).getNumElements(), Src);
782 return buildInstr(Opc: TargetOpcode::G_BUILD_VECTOR, DstOps: Res, SrcOps: TmpVec);
783}
784
785MachineInstrBuilder
786MachineIRBuilder::buildBuildVectorTrunc(const DstOp &Res,
787 ArrayRef<Register> Ops) {
788 // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
789 // we need some temporary storage for the DstOp objects. Here we use a
790 // sufficiently large SmallVector to not go through the heap.
791 SmallVector<SrcOp, 8> TmpVec(Ops);
792 if (TmpVec[0].getLLTTy(MRI: *getMRI()).getSizeInBits() ==
793 Res.getLLTTy(MRI: *getMRI()).getElementType().getSizeInBits())
794 return buildInstr(Opc: TargetOpcode::G_BUILD_VECTOR, DstOps: Res, SrcOps: TmpVec);
795 return buildInstr(Opc: TargetOpcode::G_BUILD_VECTOR_TRUNC, DstOps: Res, SrcOps: TmpVec);
796}
797
798MachineInstrBuilder MachineIRBuilder::buildShuffleSplat(const DstOp &Res,
799 const SrcOp &Src) {
800 LLT DstTy = Res.getLLTTy(MRI: *getMRI());
801 assert(Src.getLLTTy(*getMRI()) == DstTy.getElementType() &&
802 "Expected Src to match Dst elt ty");
803 auto UndefVec = buildUndef(Res: DstTy);
804 auto Zero = buildConstant(Res: LLT::integer(SizeInBits: 64), Val: 0);
805 auto InsElt = buildInsertVectorElement(Res: DstTy, Val: UndefVec, Elt: Src, Idx: Zero);
806 SmallVector<int, 16> ZeroMask(DstTy.getNumElements());
807 return buildShuffleVector(Res: DstTy, Src1: InsElt, Src2: UndefVec, Mask: ZeroMask);
808}
809
810MachineInstrBuilder MachineIRBuilder::buildSplatVector(const DstOp &Res,
811 const SrcOp &Src) {
812 assert(Src.getLLTTy(*getMRI()) == Res.getLLTTy(*getMRI()).getElementType() &&
813 "Expected Src to match Dst elt ty");
814 return buildInstr(Opc: TargetOpcode::G_SPLAT_VECTOR, DstOps: Res, SrcOps: Src);
815}
816
817MachineInstrBuilder MachineIRBuilder::buildShuffleVector(const DstOp &Res,
818 const SrcOp &Src1,
819 const SrcOp &Src2,
820 ArrayRef<int> Mask) {
821 LLT DstTy = Res.getLLTTy(MRI: *getMRI());
822 LLT Src1Ty = Src1.getLLTTy(MRI: *getMRI());
823 LLT Src2Ty = Src2.getLLTTy(MRI: *getMRI());
824 const LLT DstElemTy = DstTy.getScalarType();
825 const LLT ElemTy1 = Src1Ty.getScalarType();
826 const LLT ElemTy2 = Src2Ty.getScalarType();
827 assert(DstElemTy == ElemTy1 && DstElemTy == ElemTy2);
828 assert(Mask.size() > 1 && "Scalar G_SHUFFLE_VECTOR are not supported");
829 (void)DstElemTy;
830 (void)ElemTy1;
831 (void)ElemTy2;
832 ArrayRef<int> MaskAlloc = getMF().allocateShuffleMask(Mask);
833 return buildInstr(Opc: TargetOpcode::G_SHUFFLE_VECTOR, DstOps: {Res}, SrcOps: {Src1, Src2})
834 .addShuffleMask(Val: MaskAlloc);
835}
836
837MachineInstrBuilder
838MachineIRBuilder::buildConcatVectors(const DstOp &Res, ArrayRef<Register> Ops) {
839 // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
840 // we need some temporary storage for the DstOp objects. Here we use a
841 // sufficiently large SmallVector to not go through the heap.
842 SmallVector<SrcOp, 8> TmpVec(Ops);
843 return buildInstr(Opc: TargetOpcode::G_CONCAT_VECTORS, DstOps: Res, SrcOps: TmpVec);
844}
845
846MachineInstrBuilder MachineIRBuilder::buildInsert(const DstOp &Res,
847 const SrcOp &Src,
848 const SrcOp &Op,
849 unsigned Index) {
850 assert(Index + Op.getLLTTy(*getMRI()).getSizeInBits() <=
851 Res.getLLTTy(*getMRI()).getSizeInBits() &&
852 "insertion past the end of a register");
853
854 if (Res.getLLTTy(MRI: *getMRI()).getSizeInBits() ==
855 Op.getLLTTy(MRI: *getMRI()).getSizeInBits()) {
856 return buildCast(Dst: Res, Src: Op);
857 }
858
859 return buildInstr(Opc: TargetOpcode::G_INSERT, DstOps: Res, SrcOps: {Src, Op, uint64_t(Index)});
860}
861
862MachineInstrBuilder MachineIRBuilder::buildStepVector(const DstOp &Res,
863 unsigned Step) {
864 unsigned Bitwidth = Res.getLLTTy(MRI: *getMRI()).getElementType().getSizeInBits();
865 ConstantInt *CI = ConstantInt::get(Context&: getMF().getFunction().getContext(),
866 V: APInt(Bitwidth, Step));
867 auto StepVector = buildInstr(Opcode: TargetOpcode::G_STEP_VECTOR);
868 StepVector->setDebugLoc(DebugLoc());
869 Res.addDefToMIB(MRI&: *getMRI(), MIB&: StepVector);
870 StepVector.addCImm(Val: CI);
871 return StepVector;
872}
873
874MachineInstrBuilder MachineIRBuilder::buildVScale(const DstOp &Res,
875 unsigned MinElts) {
876
877 auto IntN = IntegerType::get(C&: getMF().getFunction().getContext(),
878 NumBits: Res.getLLTTy(MRI: *getMRI()).getScalarSizeInBits());
879 ConstantInt *CI = ConstantInt::get(Ty: IntN, V: MinElts);
880 return buildVScale(Res, MinElts: *CI);
881}
882
883MachineInstrBuilder MachineIRBuilder::buildVScale(const DstOp &Res,
884 const ConstantInt &MinElts) {
885 auto VScale = buildInstr(Opcode: TargetOpcode::G_VSCALE);
886 VScale->setDebugLoc(DebugLoc());
887 Res.addDefToMIB(MRI&: *getMRI(), MIB&: VScale);
888 VScale.addCImm(Val: &MinElts);
889 return VScale;
890}
891
892MachineInstrBuilder MachineIRBuilder::buildVScale(const DstOp &Res,
893 const APInt &MinElts) {
894 ConstantInt *CI =
895 ConstantInt::get(Context&: getMF().getFunction().getContext(), V: MinElts);
896 return buildVScale(Res, MinElts: *CI);
897}
898
899static unsigned getIntrinsicOpcode(bool HasSideEffects, bool IsConvergent) {
900 if (HasSideEffects && IsConvergent)
901 return TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS;
902 if (HasSideEffects)
903 return TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS;
904 if (IsConvergent)
905 return TargetOpcode::G_INTRINSIC_CONVERGENT;
906 return TargetOpcode::G_INTRINSIC;
907}
908
909MachineInstrBuilder
910MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
911 ArrayRef<Register> ResultRegs,
912 bool HasSideEffects, bool isConvergent) {
913 auto MIB = buildInstr(Opcode: getIntrinsicOpcode(HasSideEffects, IsConvergent: isConvergent));
914 for (Register ResultReg : ResultRegs)
915 MIB.addDef(RegNo: ResultReg);
916 MIB.addIntrinsicID(ID);
917 return MIB;
918}
919
920MachineInstrBuilder
921MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
922 ArrayRef<Register> ResultRegs) {
923 AttributeSet Attrs = Intrinsic::getFnAttributes(C&: getContext(), id: ID);
924 bool HasSideEffects = !Attrs.getMemoryEffects().doesNotAccessMemory();
925 bool isConvergent = Attrs.hasAttribute(Kind: Attribute::Convergent);
926 return buildIntrinsic(ID, ResultRegs, HasSideEffects, isConvergent);
927}
928
929MachineInstrBuilder MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
930 ArrayRef<DstOp> Results,
931 bool HasSideEffects,
932 bool isConvergent) {
933 auto MIB = buildInstr(Opcode: getIntrinsicOpcode(HasSideEffects, IsConvergent: isConvergent));
934 for (DstOp Result : Results)
935 Result.addDefToMIB(MRI&: *getMRI(), MIB);
936 MIB.addIntrinsicID(ID);
937 return MIB;
938}
939
940MachineInstrBuilder MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
941 ArrayRef<DstOp> Results) {
942 AttributeSet Attrs = Intrinsic::getFnAttributes(C&: getContext(), id: ID);
943 bool HasSideEffects = !Attrs.getMemoryEffects().doesNotAccessMemory();
944 bool isConvergent = Attrs.hasAttribute(Kind: Attribute::Convergent);
945 return buildIntrinsic(ID, Results, HasSideEffects, isConvergent);
946}
947
948MachineInstrBuilder
949MachineIRBuilder::buildTrunc(const DstOp &Res, const SrcOp &Op,
950 std::optional<unsigned> Flags) {
951 return buildInstr(Opc: TargetOpcode::G_TRUNC, DstOps: Res, SrcOps: Op, Flags);
952}
953
954MachineInstrBuilder
955MachineIRBuilder::buildFPTrunc(const DstOp &Res, const SrcOp &Op,
956 std::optional<unsigned> Flags) {
957 return buildInstr(Opc: TargetOpcode::G_FPTRUNC, DstOps: Res, SrcOps: Op, Flags);
958}
959
960MachineInstrBuilder MachineIRBuilder::buildICmp(CmpInst::Predicate Pred,
961 const DstOp &Res,
962 const SrcOp &Op0,
963 const SrcOp &Op1,
964 std::optional<unsigned> Flags) {
965 return buildInstr(Opc: TargetOpcode::G_ICMP, DstOps: Res, SrcOps: {Pred, Op0, Op1}, Flags);
966}
967
968MachineInstrBuilder MachineIRBuilder::buildFCmp(CmpInst::Predicate Pred,
969 const DstOp &Res,
970 const SrcOp &Op0,
971 const SrcOp &Op1,
972 std::optional<unsigned> Flags) {
973
974 return buildInstr(Opc: TargetOpcode::G_FCMP, DstOps: Res, SrcOps: {Pred, Op0, Op1}, Flags);
975}
976
977MachineInstrBuilder MachineIRBuilder::buildSCmp(const DstOp &Res,
978 const SrcOp &Op0,
979 const SrcOp &Op1) {
980 return buildInstr(Opc: TargetOpcode::G_SCMP, DstOps: Res, SrcOps: {Op0, Op1});
981}
982
983MachineInstrBuilder MachineIRBuilder::buildUCmp(const DstOp &Res,
984 const SrcOp &Op0,
985 const SrcOp &Op1) {
986 return buildInstr(Opc: TargetOpcode::G_UCMP, DstOps: Res, SrcOps: {Op0, Op1});
987}
988
989MachineInstrBuilder
990MachineIRBuilder::buildSelect(const DstOp &Res, const SrcOp &Tst,
991 const SrcOp &Op0, const SrcOp &Op1,
992 std::optional<unsigned> Flags) {
993
994 return buildInstr(Opc: TargetOpcode::G_SELECT, DstOps: {Res}, SrcOps: {Tst, Op0, Op1}, Flags);
995}
996
997MachineInstrBuilder MachineIRBuilder::buildInsertSubvector(const DstOp &Res,
998 const SrcOp &Src0,
999 const SrcOp &Src1,
1000 unsigned Idx) {
1001 return buildInstr(Opc: TargetOpcode::G_INSERT_SUBVECTOR, DstOps: Res,
1002 SrcOps: {Src0, Src1, uint64_t(Idx)});
1003}
1004
1005MachineInstrBuilder MachineIRBuilder::buildExtractSubvector(const DstOp &Res,
1006 const SrcOp &Src,
1007 unsigned Idx) {
1008 return buildInstr(Opc: TargetOpcode::G_EXTRACT_SUBVECTOR, DstOps: Res,
1009 SrcOps: {Src, uint64_t(Idx)});
1010}
1011
1012MachineInstrBuilder
1013MachineIRBuilder::buildInsertVectorElement(const DstOp &Res, const SrcOp &Val,
1014 const SrcOp &Elt, const SrcOp &Idx) {
1015 return buildInstr(Opc: TargetOpcode::G_INSERT_VECTOR_ELT, DstOps: Res, SrcOps: {Val, Elt, Idx});
1016}
1017
1018MachineInstrBuilder
1019MachineIRBuilder::buildExtractVectorElement(const DstOp &Res, const SrcOp &Val,
1020 const SrcOp &Idx) {
1021 return buildInstr(Opc: TargetOpcode::G_EXTRACT_VECTOR_ELT, DstOps: Res, SrcOps: {Val, Idx});
1022}
1023
1024MachineInstrBuilder MachineIRBuilder::buildAtomicCmpXchgWithSuccess(
1025 const DstOp &OldValRes, const DstOp &SuccessRes, const SrcOp &Addr,
1026 const SrcOp &CmpVal, const SrcOp &NewVal, MachineMemOperand &MMO) {
1027#ifndef NDEBUG
1028 LLT OldValResTy = OldValRes.getLLTTy(*getMRI());
1029 LLT SuccessResTy = SuccessRes.getLLTTy(*getMRI());
1030 LLT AddrTy = Addr.getLLTTy(*getMRI());
1031 LLT CmpValTy = CmpVal.getLLTTy(*getMRI());
1032 LLT NewValTy = NewVal.getLLTTy(*getMRI());
1033 assert(OldValResTy.isScalar() && "invalid operand type");
1034 assert(SuccessResTy.isScalar() && "invalid operand type");
1035 assert(AddrTy.isPointer() && "invalid operand type");
1036 assert(CmpValTy.isValid() && "invalid operand type");
1037 assert(NewValTy.isValid() && "invalid operand type");
1038 assert(OldValResTy == CmpValTy && "type mismatch");
1039 assert(OldValResTy == NewValTy && "type mismatch");
1040#endif
1041
1042 auto MIB = buildInstr(Opcode: TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS);
1043 OldValRes.addDefToMIB(MRI&: *getMRI(), MIB);
1044 SuccessRes.addDefToMIB(MRI&: *getMRI(), MIB);
1045 Addr.addSrcToMIB(MIB);
1046 CmpVal.addSrcToMIB(MIB);
1047 NewVal.addSrcToMIB(MIB);
1048 MIB.addMemOperand(MMO: &MMO);
1049 return MIB;
1050}
1051
1052MachineInstrBuilder
1053MachineIRBuilder::buildAtomicCmpXchg(const DstOp &OldValRes, const SrcOp &Addr,
1054 const SrcOp &CmpVal, const SrcOp &NewVal,
1055 MachineMemOperand &MMO) {
1056#ifndef NDEBUG
1057 LLT OldValResTy = OldValRes.getLLTTy(*getMRI());
1058 LLT AddrTy = Addr.getLLTTy(*getMRI());
1059 LLT CmpValTy = CmpVal.getLLTTy(*getMRI());
1060 LLT NewValTy = NewVal.getLLTTy(*getMRI());
1061 assert(OldValResTy.isScalar() && "invalid operand type");
1062 assert(AddrTy.isPointer() && "invalid operand type");
1063 assert(CmpValTy.isValid() && "invalid operand type");
1064 assert(NewValTy.isValid() && "invalid operand type");
1065 assert(OldValResTy == CmpValTy && "type mismatch");
1066 assert(OldValResTy == NewValTy && "type mismatch");
1067#endif
1068
1069 auto MIB = buildInstr(Opcode: TargetOpcode::G_ATOMIC_CMPXCHG);
1070 OldValRes.addDefToMIB(MRI&: *getMRI(), MIB);
1071 Addr.addSrcToMIB(MIB);
1072 CmpVal.addSrcToMIB(MIB);
1073 NewVal.addSrcToMIB(MIB);
1074 MIB.addMemOperand(MMO: &MMO);
1075 return MIB;
1076}
1077
1078MachineInstrBuilder MachineIRBuilder::buildAtomicRMW(
1079 unsigned Opcode, const DstOp &OldValRes,
1080 const SrcOp &Addr, const SrcOp &Val,
1081 MachineMemOperand &MMO) {
1082
1083#ifndef NDEBUG
1084 LLT OldValResTy = OldValRes.getLLTTy(*getMRI());
1085 LLT AddrTy = Addr.getLLTTy(*getMRI());
1086 LLT ValTy = Val.getLLTTy(*getMRI());
1087 assert(AddrTy.isPointer() && "invalid operand type");
1088 assert(ValTy.isValid() && "invalid operand type");
1089 assert(OldValResTy == ValTy && "type mismatch");
1090 assert(MMO.isAtomic() && "not atomic mem operand");
1091#endif
1092
1093 auto MIB = buildInstr(Opcode);
1094 OldValRes.addDefToMIB(MRI&: *getMRI(), MIB);
1095 Addr.addSrcToMIB(MIB);
1096 Val.addSrcToMIB(MIB);
1097 MIB.addMemOperand(MMO: &MMO);
1098 return MIB;
1099}
1100
1101MachineInstrBuilder
1102MachineIRBuilder::buildAtomicRMWXchg(Register OldValRes, Register Addr,
1103 Register Val, MachineMemOperand &MMO) {
1104 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_XCHG, OldValRes, Addr, Val,
1105 MMO);
1106}
1107MachineInstrBuilder
1108MachineIRBuilder::buildAtomicRMWAdd(Register OldValRes, Register Addr,
1109 Register Val, MachineMemOperand &MMO) {
1110 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_ADD, OldValRes, Addr, Val,
1111 MMO);
1112}
1113MachineInstrBuilder
1114MachineIRBuilder::buildAtomicRMWSub(Register OldValRes, Register Addr,
1115 Register Val, MachineMemOperand &MMO) {
1116 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_SUB, OldValRes, Addr, Val,
1117 MMO);
1118}
1119MachineInstrBuilder
1120MachineIRBuilder::buildAtomicRMWAnd(Register OldValRes, Register Addr,
1121 Register Val, MachineMemOperand &MMO) {
1122 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_AND, OldValRes, Addr, Val,
1123 MMO);
1124}
1125MachineInstrBuilder
1126MachineIRBuilder::buildAtomicRMWNand(Register OldValRes, Register Addr,
1127 Register Val, MachineMemOperand &MMO) {
1128 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_NAND, OldValRes, Addr, Val,
1129 MMO);
1130}
1131MachineInstrBuilder MachineIRBuilder::buildAtomicRMWOr(Register OldValRes,
1132 Register Addr,
1133 Register Val,
1134 MachineMemOperand &MMO) {
1135 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_OR, OldValRes, Addr, Val,
1136 MMO);
1137}
1138MachineInstrBuilder
1139MachineIRBuilder::buildAtomicRMWXor(Register OldValRes, Register Addr,
1140 Register Val, MachineMemOperand &MMO) {
1141 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_XOR, OldValRes, Addr, Val,
1142 MMO);
1143}
1144MachineInstrBuilder
1145MachineIRBuilder::buildAtomicRMWMax(Register OldValRes, Register Addr,
1146 Register Val, MachineMemOperand &MMO) {
1147 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_MAX, OldValRes, Addr, Val,
1148 MMO);
1149}
1150MachineInstrBuilder
1151MachineIRBuilder::buildAtomicRMWMin(Register OldValRes, Register Addr,
1152 Register Val, MachineMemOperand &MMO) {
1153 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_MIN, OldValRes, Addr, Val,
1154 MMO);
1155}
1156MachineInstrBuilder
1157MachineIRBuilder::buildAtomicRMWUmax(Register OldValRes, Register Addr,
1158 Register Val, MachineMemOperand &MMO) {
1159 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_UMAX, OldValRes, Addr, Val,
1160 MMO);
1161}
1162MachineInstrBuilder
1163MachineIRBuilder::buildAtomicRMWUmin(Register OldValRes, Register Addr,
1164 Register Val, MachineMemOperand &MMO) {
1165 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_UMIN, OldValRes, Addr, Val,
1166 MMO);
1167}
1168
1169MachineInstrBuilder
1170MachineIRBuilder::buildAtomicRMWFAdd(
1171 const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val,
1172 MachineMemOperand &MMO) {
1173 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_FADD, OldValRes, Addr, Val,
1174 MMO);
1175}
1176
1177MachineInstrBuilder
1178MachineIRBuilder::buildAtomicRMWFSub(const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val,
1179 MachineMemOperand &MMO) {
1180 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_FSUB, OldValRes, Addr, Val,
1181 MMO);
1182}
1183
1184MachineInstrBuilder
1185MachineIRBuilder::buildAtomicRMWFMax(const DstOp &OldValRes, const SrcOp &Addr,
1186 const SrcOp &Val, MachineMemOperand &MMO) {
1187 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_FMAX, OldValRes, Addr, Val,
1188 MMO);
1189}
1190
1191MachineInstrBuilder
1192MachineIRBuilder::buildAtomicRMWFMin(const DstOp &OldValRes, const SrcOp &Addr,
1193 const SrcOp &Val, MachineMemOperand &MMO) {
1194 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_FMIN, OldValRes, Addr, Val,
1195 MMO);
1196}
1197
1198MachineInstrBuilder
1199MachineIRBuilder::buildAtomicRMWFMaximum(const DstOp &OldValRes,
1200 const SrcOp &Addr, const SrcOp &Val,
1201 MachineMemOperand &MMO) {
1202 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_FMAXIMUM, OldValRes, Addr,
1203 Val, MMO);
1204}
1205
1206MachineInstrBuilder
1207MachineIRBuilder::buildAtomicRMWFMinimum(const DstOp &OldValRes,
1208 const SrcOp &Addr, const SrcOp &Val,
1209 MachineMemOperand &MMO) {
1210 return buildAtomicRMW(Opcode: TargetOpcode::G_ATOMICRMW_FMINIMUM, OldValRes, Addr,
1211 Val, MMO);
1212}
1213
1214MachineInstrBuilder
1215MachineIRBuilder::buildFence(unsigned Ordering, unsigned Scope) {
1216 return buildInstr(Opcode: TargetOpcode::G_FENCE)
1217 .addImm(Val: Ordering)
1218 .addImm(Val: Scope);
1219}
1220
1221MachineInstrBuilder MachineIRBuilder::buildPrefetch(const SrcOp &Addr,
1222 unsigned RW,
1223 unsigned Locality,
1224 unsigned CacheType,
1225 MachineMemOperand &MMO) {
1226 auto MIB = buildInstr(Opcode: TargetOpcode::G_PREFETCH);
1227 Addr.addSrcToMIB(MIB);
1228 MIB.addImm(Val: RW).addImm(Val: Locality).addImm(Val: CacheType);
1229 MIB.addMemOperand(MMO: &MMO);
1230 return MIB;
1231}
1232
1233MachineInstrBuilder
1234MachineIRBuilder::buildBlockAddress(Register Res, const BlockAddress *BA) {
1235#ifndef NDEBUG
1236 assert(getMRI()->getType(Res).isPointer() && "invalid res type");
1237#endif
1238
1239 return buildInstr(Opcode: TargetOpcode::G_BLOCK_ADDR).addDef(RegNo: Res).addBlockAddress(BA);
1240}
1241
1242void MachineIRBuilder::validateTruncExt(const LLT DstTy, const LLT SrcTy,
1243 bool IsExtend) {
1244#ifndef NDEBUG
1245 if (DstTy.isVector()) {
1246 assert(SrcTy.isVector() && "mismatched cast between vector and non-vector");
1247 assert(SrcTy.getElementCount() == DstTy.getElementCount() &&
1248 "different number of elements in a trunc/ext");
1249 } else
1250 assert(DstTy.isScalar() && SrcTy.isScalar() && "invalid extend/trunc");
1251
1252 if (IsExtend)
1253 assert(TypeSize::isKnownGT(DstTy.getSizeInBits(), SrcTy.getSizeInBits()) &&
1254 "invalid narrowing extend");
1255 else
1256 assert(TypeSize::isKnownLT(DstTy.getSizeInBits(), SrcTy.getSizeInBits()) &&
1257 "invalid widening trunc");
1258#endif
1259}
1260
1261void MachineIRBuilder::validateSelectOp(const LLT ResTy, const LLT TstTy,
1262 const LLT Op0Ty, const LLT Op1Ty) {
1263#ifndef NDEBUG
1264 assert((ResTy.isScalar() || ResTy.isVector() || ResTy.isPointer()) &&
1265 "invalid operand type");
1266 assert((ResTy == Op0Ty && ResTy == Op1Ty) && "type mismatch");
1267 if (ResTy.isScalar() || ResTy.isPointer())
1268 assert(TstTy.isScalar() && "type mismatch");
1269 else
1270 assert((TstTy.isScalar() ||
1271 (TstTy.isVector() &&
1272 TstTy.getElementCount() == Op0Ty.getElementCount())) &&
1273 "type mismatch");
1274#endif
1275}
1276
1277MachineInstrBuilder
1278MachineIRBuilder::buildInstr(unsigned Opc, ArrayRef<DstOp> DstOps,
1279 ArrayRef<SrcOp> SrcOps,
1280 std::optional<unsigned> Flags) {
1281 switch (Opc) {
1282 default:
1283 break;
1284 case TargetOpcode::G_SELECT: {
1285 assert(DstOps.size() == 1 && "Invalid select");
1286 assert(SrcOps.size() == 3 && "Invalid select");
1287 validateSelectOp(
1288 ResTy: DstOps[0].getLLTTy(MRI: *getMRI()), TstTy: SrcOps[0].getLLTTy(MRI: *getMRI()),
1289 Op0Ty: SrcOps[1].getLLTTy(MRI: *getMRI()), Op1Ty: SrcOps[2].getLLTTy(MRI: *getMRI()));
1290 break;
1291 }
1292 case TargetOpcode::G_FNEG:
1293 case TargetOpcode::G_ABS:
1294 // All these are unary ops.
1295 assert(DstOps.size() == 1 && "Invalid Dst");
1296 assert(SrcOps.size() == 1 && "Invalid Srcs");
1297 validateUnaryOp(Res: DstOps[0].getLLTTy(MRI: *getMRI()),
1298 Op0: SrcOps[0].getLLTTy(MRI: *getMRI()));
1299 break;
1300 case TargetOpcode::G_ADD:
1301 case TargetOpcode::G_AND:
1302 case TargetOpcode::G_MUL:
1303 case TargetOpcode::G_OR:
1304 case TargetOpcode::G_SUB:
1305 case TargetOpcode::G_XOR:
1306 case TargetOpcode::G_UDIV:
1307 case TargetOpcode::G_SDIV:
1308 case TargetOpcode::G_UREM:
1309 case TargetOpcode::G_SREM:
1310 case TargetOpcode::G_SMIN:
1311 case TargetOpcode::G_SMAX:
1312 case TargetOpcode::G_UMIN:
1313 case TargetOpcode::G_UMAX:
1314 case TargetOpcode::G_UADDSAT:
1315 case TargetOpcode::G_SADDSAT:
1316 case TargetOpcode::G_USUBSAT:
1317 case TargetOpcode::G_SSUBSAT: {
1318 // All these are binary ops.
1319 assert(DstOps.size() == 1 && "Invalid Dst");
1320 assert(SrcOps.size() == 2 && "Invalid Srcs");
1321 validateBinaryOp(Res: DstOps[0].getLLTTy(MRI: *getMRI()),
1322 Op0: SrcOps[0].getLLTTy(MRI: *getMRI()),
1323 Op1: SrcOps[1].getLLTTy(MRI: *getMRI()));
1324 break;
1325 }
1326 case TargetOpcode::G_SHL:
1327 case TargetOpcode::G_ASHR:
1328 case TargetOpcode::G_LSHR:
1329 case TargetOpcode::G_USHLSAT:
1330 case TargetOpcode::G_SSHLSAT: {
1331 assert(DstOps.size() == 1 && "Invalid Dst");
1332 assert(SrcOps.size() == 2 && "Invalid Srcs");
1333 validateShiftOp(Res: DstOps[0].getLLTTy(MRI: *getMRI()),
1334 Op0: SrcOps[0].getLLTTy(MRI: *getMRI()),
1335 Op1: SrcOps[1].getLLTTy(MRI: *getMRI()));
1336 break;
1337 }
1338 case TargetOpcode::G_SEXT:
1339 case TargetOpcode::G_ZEXT:
1340 case TargetOpcode::G_ANYEXT:
1341 assert(DstOps.size() == 1 && "Invalid Dst");
1342 assert(SrcOps.size() == 1 && "Invalid Srcs");
1343 validateTruncExt(DstTy: DstOps[0].getLLTTy(MRI: *getMRI()),
1344 SrcTy: SrcOps[0].getLLTTy(MRI: *getMRI()), IsExtend: true);
1345 break;
1346 case TargetOpcode::G_TRUNC:
1347 case TargetOpcode::G_FPTRUNC: {
1348 assert(DstOps.size() == 1 && "Invalid Dst");
1349 assert(SrcOps.size() == 1 && "Invalid Srcs");
1350 validateTruncExt(DstTy: DstOps[0].getLLTTy(MRI: *getMRI()),
1351 SrcTy: SrcOps[0].getLLTTy(MRI: *getMRI()), IsExtend: false);
1352 break;
1353 }
1354 case TargetOpcode::G_BITCAST: {
1355 assert(DstOps.size() == 1 && "Invalid Dst");
1356 assert(SrcOps.size() == 1 && "Invalid Srcs");
1357 assert(DstOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1358 SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() && "invalid bitcast");
1359 break;
1360 }
1361 case TargetOpcode::COPY:
1362 assert(DstOps.size() == 1 && "Invalid Dst");
1363 // If the caller wants to add a subreg source it has to be done separately
1364 // so we may not have any SrcOps at this point yet.
1365 break;
1366 case TargetOpcode::G_FCMP:
1367 case TargetOpcode::G_ICMP: {
1368 assert(DstOps.size() == 1 && "Invalid Dst Operands");
1369 assert(SrcOps.size() == 3 && "Invalid Src Operands");
1370 // For F/ICMP, the first src operand is the predicate, followed by
1371 // the two comparands.
1372 assert(SrcOps[0].getSrcOpKind() == SrcOp::SrcType::Ty_Predicate &&
1373 "Expecting predicate");
1374 assert([&]() -> bool {
1375 CmpInst::Predicate Pred = SrcOps[0].getPredicate();
1376 return Opc == TargetOpcode::G_ICMP ? CmpInst::isIntPredicate(Pred)
1377 : CmpInst::isFPPredicate(Pred);
1378 }() && "Invalid predicate");
1379 assert(SrcOps[1].getLLTTy(*getMRI()) == SrcOps[2].getLLTTy(*getMRI()) &&
1380 "Type mismatch");
1381 assert([&]() -> bool {
1382 LLT Op0Ty = SrcOps[1].getLLTTy(*getMRI());
1383 LLT DstTy = DstOps[0].getLLTTy(*getMRI());
1384 if (Op0Ty.isScalar() || Op0Ty.isPointer())
1385 return DstTy.isScalar();
1386 else
1387 return DstTy.isVector() &&
1388 DstTy.getElementCount() == Op0Ty.getElementCount();
1389 }() && "Type Mismatch");
1390 break;
1391 }
1392 case TargetOpcode::G_UNMERGE_VALUES: {
1393 assert(!DstOps.empty() && "Invalid trivial sequence");
1394 assert(SrcOps.size() == 1 && "Invalid src for Unmerge");
1395 assert(llvm::all_of(DstOps,
1396 [&, this](const DstOp &Op) {
1397 return Op.getLLTTy(*getMRI()) ==
1398 DstOps[0].getLLTTy(*getMRI());
1399 }) &&
1400 "type mismatch in output list");
1401 assert((TypeSize::ScalarTy)DstOps.size() *
1402 DstOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1403 SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1404 "input operands do not cover output register");
1405 break;
1406 }
1407 case TargetOpcode::G_MERGE_VALUES: {
1408 assert(SrcOps.size() >= 2 && "invalid trivial sequence");
1409 assert(DstOps.size() == 1 && "Invalid Dst");
1410 assert(llvm::all_of(SrcOps,
1411 [&, this](const SrcOp &Op) {
1412 return Op.getLLTTy(*getMRI()) ==
1413 SrcOps[0].getLLTTy(*getMRI());
1414 }) &&
1415 "type mismatch in input list");
1416 assert((TypeSize::ScalarTy)SrcOps.size() *
1417 SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1418 DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1419 "input operands do not cover output register");
1420 assert(!DstOps[0].getLLTTy(*getMRI()).isVector() &&
1421 "vectors should be built with G_CONCAT_VECTOR or G_BUILD_VECTOR");
1422 break;
1423 }
1424 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
1425 assert(DstOps.size() == 1 && "Invalid Dst size");
1426 assert(SrcOps.size() == 2 && "Invalid Src size");
1427 assert(SrcOps[0].getLLTTy(*getMRI()).isVector() && "Invalid operand type");
1428 assert((DstOps[0].getLLTTy(*getMRI()).isScalar() ||
1429 DstOps[0].getLLTTy(*getMRI()).isPointer()) &&
1430 "Invalid operand type");
1431 assert(SrcOps[1].getLLTTy(*getMRI()).isScalar() && "Invalid operand type");
1432 assert(SrcOps[0].getLLTTy(*getMRI()).getElementType() ==
1433 DstOps[0].getLLTTy(*getMRI()) &&
1434 "Type mismatch");
1435 break;
1436 }
1437 case TargetOpcode::G_INSERT_VECTOR_ELT: {
1438 assert(DstOps.size() == 1 && "Invalid dst size");
1439 assert(SrcOps.size() == 3 && "Invalid src size");
1440 assert(DstOps[0].getLLTTy(*getMRI()).isVector() &&
1441 SrcOps[0].getLLTTy(*getMRI()).isVector() && "Invalid operand type");
1442 assert(DstOps[0].getLLTTy(*getMRI()).getElementType() ==
1443 SrcOps[1].getLLTTy(*getMRI()) &&
1444 "Type mismatch");
1445 assert(SrcOps[2].getLLTTy(*getMRI()).isScalar() && "Invalid index");
1446 assert(DstOps[0].getLLTTy(*getMRI()).getElementCount() ==
1447 SrcOps[0].getLLTTy(*getMRI()).getElementCount() &&
1448 "Type mismatch");
1449 break;
1450 }
1451 case TargetOpcode::G_INSERT_SUBVECTOR: {
1452 assert(DstOps.size() == 1 && "Invalid Dst");
1453 assert(SrcOps.size() == 3 && "Invalid Srcs");
1454 [[maybe_unused]] LLT DstTy = DstOps[0].getLLTTy(MRI: *getMRI());
1455 [[maybe_unused]] LLT BigVecTy = SrcOps[0].getLLTTy(MRI: *getMRI());
1456 [[maybe_unused]] LLT SubVecTy = SrcOps[1].getLLTTy(MRI: *getMRI());
1457 assert(DstTy == BigVecTy &&
1458 "Dest and insert subvector source types must match!");
1459 assert(DstTy.isVector() && SubVecTy.isVector() &&
1460 "Insert subvector VTs must be vectors!");
1461 assert(DstTy.getElementType() == SubVecTy.getElementType() &&
1462 "Insert subvector VTs must have the same element type!");
1463 assert((DstTy.isScalable() || !SubVecTy.isScalable()) &&
1464 "Cannot insert a scalable vector into a fixed length vector!");
1465 assert((DstTy.isScalable() != SubVecTy.isScalable() ||
1466 DstTy.getElementCount().getKnownMinValue() >=
1467 SubVecTy.getElementCount().getKnownMinValue()) &&
1468 "Insert subvector must be from smaller vector to larger vector!");
1469 assert(SrcOps[2].getSrcOpKind() == SrcOp::SrcType::Ty_Imm &&
1470 "Insert subvector index must be constant");
1471 assert((DstTy.isScalable() != SubVecTy.isScalable() ||
1472 (SubVecTy.getElementCount().getKnownMinValue() +
1473 (uint64_t)SrcOps[2].getImm()) <=
1474 DstTy.getElementCount().getKnownMinValue()) &&
1475 "Insert subvector overflow!");
1476 assert((uint64_t)SrcOps[2].getImm() %
1477 SubVecTy.getElementCount().getKnownMinValue() ==
1478 0 &&
1479 "Insert index is not a multiple of the subvector length");
1480 break;
1481 }
1482 case TargetOpcode::G_EXTRACT_SUBVECTOR: {
1483 assert(DstOps.size() == 1 && "Invalid Dst");
1484 assert(SrcOps.size() == 2 && "Invalid Srcs");
1485 [[maybe_unused]] LLT DstTy = DstOps[0].getLLTTy(MRI: *getMRI());
1486 [[maybe_unused]] LLT SrcVecTy = SrcOps[0].getLLTTy(MRI: *getMRI());
1487 assert(DstTy.isVector() && SrcVecTy.isVector() &&
1488 "Extract subvector VTs must be vectors!");
1489 assert(DstTy.getElementType() == SrcVecTy.getElementType() &&
1490 "Extract subvector VTs must have the same element type!");
1491 assert((!DstTy.isScalable() || SrcVecTy.isScalable()) &&
1492 "Cannot extract a scalable vector from a fixed length vector!");
1493 assert((DstTy.isScalable() != SrcVecTy.isScalable() ||
1494 DstTy.getElementCount().getKnownMinValue() <=
1495 SrcVecTy.getElementCount().getKnownMinValue()) &&
1496 "Extract subvector must be from larger vector to smaller vector!");
1497 assert(SrcOps[1].getSrcOpKind() == SrcOp::SrcType::Ty_Imm &&
1498 "Extract subvector index must be a constant");
1499 assert((DstTy.isScalable() != SrcVecTy.isScalable() ||
1500 (DstTy.getElementCount().getKnownMinValue() +
1501 (uint64_t)SrcOps[1].getImm()) <=
1502 SrcVecTy.getElementCount().getKnownMinValue()) &&
1503 "Extract subvector overflow!");
1504 assert((uint64_t)SrcOps[1].getImm() %
1505 DstTy.getElementCount().getKnownMinValue() ==
1506 0 &&
1507 "Extract index is not a multiple of the output vector length");
1508 break;
1509 }
1510 case TargetOpcode::G_BUILD_VECTOR: {
1511 assert((!SrcOps.empty() || SrcOps.size() < 2) &&
1512 "Must have at least 2 operands");
1513 assert(DstOps.size() == 1 && "Invalid DstOps");
1514 assert(DstOps[0].getLLTTy(*getMRI()).isVector() &&
1515 "Res type must be a vector");
1516 assert(llvm::all_of(SrcOps,
1517 [&, this](const SrcOp &Op) {
1518 return Op.getLLTTy(*getMRI()) ==
1519 SrcOps[0].getLLTTy(*getMRI());
1520 }) &&
1521 "type mismatch in input list");
1522 assert((TypeSize::ScalarTy)SrcOps.size() *
1523 SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1524 DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1525 "input scalars do not exactly cover the output vector register");
1526 break;
1527 }
1528 case TargetOpcode::G_BUILD_VECTOR_TRUNC: {
1529 assert((!SrcOps.empty() || SrcOps.size() < 2) &&
1530 "Must have at least 2 operands");
1531 assert(DstOps.size() == 1 && "Invalid DstOps");
1532 assert(DstOps[0].getLLTTy(*getMRI()).isVector() &&
1533 "Res type must be a vector");
1534 assert(llvm::all_of(SrcOps,
1535 [&, this](const SrcOp &Op) {
1536 return Op.getLLTTy(*getMRI()) ==
1537 SrcOps[0].getLLTTy(*getMRI());
1538 }) &&
1539 "type mismatch in input list");
1540 break;
1541 }
1542 case TargetOpcode::G_CONCAT_VECTORS: {
1543 assert(DstOps.size() == 1 && "Invalid DstOps");
1544 assert((!SrcOps.empty() || SrcOps.size() < 2) &&
1545 "Must have at least 2 operands");
1546 assert(llvm::all_of(SrcOps,
1547 [&, this](const SrcOp &Op) {
1548 return (Op.getLLTTy(*getMRI()).isVector() &&
1549 Op.getLLTTy(*getMRI()) ==
1550 SrcOps[0].getLLTTy(*getMRI()));
1551 }) &&
1552 "type mismatch in input list");
1553 assert((TypeSize::ScalarTy)SrcOps.size() *
1554 SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1555 DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1556 "input vectors do not exactly cover the output vector register");
1557 break;
1558 }
1559 case TargetOpcode::G_UADDE: {
1560 assert(DstOps.size() == 2 && "Invalid no of dst operands");
1561 assert(SrcOps.size() == 3 && "Invalid no of src operands");
1562 assert(DstOps[0].getLLTTy(*getMRI()).isScalar() && "Invalid operand");
1563 assert((DstOps[0].getLLTTy(*getMRI()) == SrcOps[0].getLLTTy(*getMRI())) &&
1564 (DstOps[0].getLLTTy(*getMRI()) == SrcOps[1].getLLTTy(*getMRI())) &&
1565 "Invalid operand");
1566 assert(DstOps[1].getLLTTy(*getMRI()).isScalar() && "Invalid operand");
1567 assert(DstOps[1].getLLTTy(*getMRI()) == SrcOps[2].getLLTTy(*getMRI()) &&
1568 "type mismatch");
1569 break;
1570 }
1571 }
1572
1573 auto MIB = buildInstr(Opcode: Opc);
1574 for (const DstOp &Op : DstOps)
1575 Op.addDefToMIB(MRI&: *getMRI(), MIB);
1576 for (const SrcOp &Op : SrcOps)
1577 Op.addSrcToMIB(MIB);
1578 if (Flags)
1579 MIB->setFlags(*Flags);
1580 return MIB;
1581}
1582