1//==--- InstrEmitter.cpp - Emit MachineInstrs for the SelectionDAG class ---==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the Emit routines for the SelectionDAG class, which creates
10// MachineInstrs based on the decisions of the SelectionDAG instruction
11// selection.
12//
13//===----------------------------------------------------------------------===//
14
15#include "InstrEmitter.h"
16#include "SDNodeDbgValue.h"
17#include "llvm/BinaryFormat/Dwarf.h"
18#include "llvm/CodeGen/MachineConstantPool.h"
19#include "llvm/CodeGen/MachineFunction.h"
20#include "llvm/CodeGen/MachineInstrBuilder.h"
21#include "llvm/CodeGen/MachineRegisterInfo.h"
22#include "llvm/CodeGen/StackMaps.h"
23#include "llvm/CodeGen/TargetInstrInfo.h"
24#include "llvm/CodeGen/TargetLowering.h"
25#include "llvm/CodeGen/TargetSubtargetInfo.h"
26#include "llvm/IR/DebugInfoMetadata.h"
27#include "llvm/IR/PseudoProbe.h"
28#include "llvm/Support/ErrorHandling.h"
29#include "llvm/Target/TargetMachine.h"
30using namespace llvm;
31
32#define DEBUG_TYPE "instr-emitter"
33
34/// MinRCSize - Smallest register class we allow when constraining virtual
35/// registers. If satisfying all register class constraints would require
36/// using a smaller register class, emit a COPY to a new virtual register
37/// instead.
38const unsigned MinRCSize = 4;
39
40/// CountResults - The results of target nodes have register or immediate
41/// operands first, then an optional chain, and optional glue operands (which do
42/// not go into the resulting MachineInstr).
43unsigned InstrEmitter::CountResults(SDNode *Node) {
44 unsigned N = Node->getNumValues();
45 while (N && Node->getValueType(ResNo: N - 1) == MVT::Glue)
46 --N;
47 if (N && Node->getValueType(ResNo: N - 1) == MVT::Other)
48 --N; // Skip over chain result.
49 return N;
50}
51
52/// countOperands - The inputs to target nodes have any actual inputs first,
53/// followed by an optional chain operand, then an optional glue operand.
54/// Compute the number of actual operands that will go into the resulting
55/// MachineInstr.
56///
57/// Also count physreg RegisterSDNode and RegisterMaskSDNode operands preceding
58/// the chain and glue. These operands may be implicit on the machine instr.
59static unsigned countOperands(SDNode *Node, unsigned NumExpUses,
60 unsigned &NumImpUses) {
61 unsigned N = Node->getNumOperands();
62 while (N && Node->getOperand(Num: N - 1).getValueType() == MVT::Glue)
63 --N;
64 if (N && Node->getOperand(Num: N - 1).getValueType() == MVT::Other)
65 --N; // Ignore chain if it exists.
66
67 // Count RegisterSDNode and RegisterMaskSDNode operands for NumImpUses.
68 NumImpUses = N - NumExpUses;
69 for (unsigned I = N; I > NumExpUses; --I) {
70 if (isa<RegisterMaskSDNode>(Val: Node->getOperand(Num: I - 1)))
71 continue;
72 if (RegisterSDNode *RN = dyn_cast<RegisterSDNode>(Val: Node->getOperand(Num: I - 1)))
73 if (RN->getReg().isPhysical())
74 continue;
75 NumImpUses = N - I;
76 break;
77 }
78
79 return N;
80}
81
82/// EmitCopyFromReg - Generate machine code for an CopyFromReg node or an
83/// implicit physical register output.
84void InstrEmitter::EmitCopyFromReg(SDNode *Node, unsigned ResNo, bool IsClone,
85 Register SrcReg,
86 DenseMap<SDValue, Register> &VRBaseMap) {
87 Register VRBase;
88 if (SrcReg.isVirtual()) {
89 // Just use the input register directly!
90 SDValue Op(Node, ResNo);
91 if (IsClone)
92 VRBaseMap.erase(Val: Op);
93 bool isNew = VRBaseMap.insert(KV: std::make_pair(x&: Op, y&: SrcReg)).second;
94 (void)isNew; // Silence compiler warning.
95 assert(isNew && "Node emitted out of order - early");
96 return;
97 }
98
99 // If the node is only used by a CopyToReg and the dest reg is a vreg, use
100 // the CopyToReg'd destination register instead of creating a new vreg.
101 bool MatchReg = true;
102 const TargetRegisterClass *UseRC = nullptr;
103 MVT VT = Node->getSimpleValueType(ResNo);
104
105 // Stick to the preferred register classes for legal types.
106 if (TLI->isTypeLegal(VT))
107 UseRC = TLI->getRegClassFor(VT, isDivergent: Node->isDivergent());
108
109 for (SDNode *User : Node->uses()) {
110 bool Match = true;
111 if (User->getOpcode() == ISD::CopyToReg &&
112 User->getOperand(Num: 2).getNode() == Node &&
113 User->getOperand(Num: 2).getResNo() == ResNo) {
114 Register DestReg = cast<RegisterSDNode>(Val: User->getOperand(Num: 1))->getReg();
115 if (DestReg.isVirtual()) {
116 VRBase = DestReg;
117 Match = false;
118 } else if (DestReg != SrcReg)
119 Match = false;
120 } else {
121 for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) {
122 SDValue Op = User->getOperand(Num: i);
123 if (Op.getNode() != Node || Op.getResNo() != ResNo)
124 continue;
125 MVT VT = Node->getSimpleValueType(ResNo: Op.getResNo());
126 if (VT == MVT::Other || VT == MVT::Glue)
127 continue;
128 Match = false;
129 if (User->isMachineOpcode()) {
130 const MCInstrDesc &II = TII->get(Opcode: User->getMachineOpcode());
131 const TargetRegisterClass *RC = nullptr;
132 if (i + II.getNumDefs() < II.getNumOperands()) {
133 RC = TRI->getAllocatableClass(
134 RC: TII->getRegClass(MCID: II, OpNum: i + II.getNumDefs(), TRI, MF: *MF));
135 }
136 if (!UseRC)
137 UseRC = RC;
138 else if (RC) {
139 const TargetRegisterClass *ComRC =
140 TRI->getCommonSubClass(A: UseRC, B: RC);
141 // If multiple uses expect disjoint register classes, we emit
142 // copies in AddRegisterOperand.
143 if (ComRC)
144 UseRC = ComRC;
145 }
146 }
147 }
148 }
149 MatchReg &= Match;
150 if (VRBase)
151 break;
152 }
153
154 const TargetRegisterClass *SrcRC = nullptr, *DstRC = nullptr;
155 SrcRC = TRI->getMinimalPhysRegClass(Reg: SrcReg, VT);
156
157 // Figure out the register class to create for the destreg.
158 if (VRBase) {
159 DstRC = MRI->getRegClass(Reg: VRBase);
160 } else if (UseRC) {
161 assert(TRI->isTypeLegalForClass(*UseRC, VT) &&
162 "Incompatible phys register def and uses!");
163 DstRC = UseRC;
164 } else
165 DstRC = SrcRC;
166
167 // If all uses are reading from the src physical register and copying the
168 // register is either impossible or very expensive, then don't create a copy.
169 if (MatchReg && SrcRC->getCopyCost() < 0) {
170 VRBase = SrcReg;
171 } else {
172 // Create the reg, emit the copy.
173 VRBase = MRI->createVirtualRegister(RegClass: DstRC);
174 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Node->getDebugLoc(), MCID: TII->get(Opcode: TargetOpcode::COPY),
175 DestReg: VRBase).addReg(RegNo: SrcReg);
176 }
177
178 SDValue Op(Node, ResNo);
179 if (IsClone)
180 VRBaseMap.erase(Val: Op);
181 bool isNew = VRBaseMap.insert(KV: std::make_pair(x&: Op, y&: VRBase)).second;
182 (void)isNew; // Silence compiler warning.
183 assert(isNew && "Node emitted out of order - early");
184}
185
186void InstrEmitter::CreateVirtualRegisters(SDNode *Node,
187 MachineInstrBuilder &MIB,
188 const MCInstrDesc &II,
189 bool IsClone, bool IsCloned,
190 DenseMap<SDValue, Register> &VRBaseMap) {
191 assert(Node->getMachineOpcode() != TargetOpcode::IMPLICIT_DEF &&
192 "IMPLICIT_DEF should have been handled as a special case elsewhere!");
193
194 unsigned NumResults = CountResults(Node);
195 bool HasVRegVariadicDefs = !MF->getTarget().usesPhysRegsForValues() &&
196 II.isVariadic() && II.variadicOpsAreDefs();
197 unsigned NumVRegs = HasVRegVariadicDefs ? NumResults : II.getNumDefs();
198 if (Node->getMachineOpcode() == TargetOpcode::STATEPOINT)
199 NumVRegs = NumResults;
200 for (unsigned i = 0; i < NumVRegs; ++i) {
201 // If the specific node value is only used by a CopyToReg and the dest reg
202 // is a vreg in the same register class, use the CopyToReg'd destination
203 // register instead of creating a new vreg.
204 Register VRBase;
205 const TargetRegisterClass *RC =
206 TRI->getAllocatableClass(RC: TII->getRegClass(MCID: II, OpNum: i, TRI, MF: *MF));
207 // Always let the value type influence the used register class. The
208 // constraints on the instruction may be too lax to represent the value
209 // type correctly. For example, a 64-bit float (X86::FR64) can't live in
210 // the 32-bit float super-class (X86::FR32).
211 if (i < NumResults && TLI->isTypeLegal(VT: Node->getSimpleValueType(ResNo: i))) {
212 const TargetRegisterClass *VTRC = TLI->getRegClassFor(
213 VT: Node->getSimpleValueType(ResNo: i),
214 isDivergent: (Node->isDivergent() || (RC && TRI->isDivergentRegClass(RC))));
215 if (RC)
216 VTRC = TRI->getCommonSubClass(A: RC, B: VTRC);
217 if (VTRC)
218 RC = VTRC;
219 }
220
221 if (!II.operands().empty() && II.operands()[i].isOptionalDef()) {
222 // Optional def must be a physical register.
223 VRBase = cast<RegisterSDNode>(Val: Node->getOperand(Num: i-NumResults))->getReg();
224 assert(VRBase.isPhysical());
225 MIB.addReg(RegNo: VRBase, flags: RegState::Define);
226 }
227
228 if (!VRBase && !IsClone && !IsCloned)
229 for (SDNode *User : Node->uses()) {
230 if (User->getOpcode() == ISD::CopyToReg &&
231 User->getOperand(Num: 2).getNode() == Node &&
232 User->getOperand(Num: 2).getResNo() == i) {
233 Register Reg = cast<RegisterSDNode>(Val: User->getOperand(Num: 1))->getReg();
234 if (Reg.isVirtual()) {
235 const TargetRegisterClass *RegRC = MRI->getRegClass(Reg);
236 if (RegRC == RC) {
237 VRBase = Reg;
238 MIB.addReg(RegNo: VRBase, flags: RegState::Define);
239 break;
240 }
241 }
242 }
243 }
244
245 // Create the result registers for this node and add the result regs to
246 // the machine instruction.
247 if (VRBase == 0) {
248 assert(RC && "Isn't a register operand!");
249 VRBase = MRI->createVirtualRegister(RegClass: RC);
250 MIB.addReg(RegNo: VRBase, flags: RegState::Define);
251 }
252
253 // If this def corresponds to a result of the SDNode insert the VRBase into
254 // the lookup map.
255 if (i < NumResults) {
256 SDValue Op(Node, i);
257 if (IsClone)
258 VRBaseMap.erase(Val: Op);
259 bool isNew = VRBaseMap.insert(KV: std::make_pair(x&: Op, y&: VRBase)).second;
260 (void)isNew; // Silence compiler warning.
261 assert(isNew && "Node emitted out of order - early");
262 }
263 }
264}
265
266/// getVR - Return the virtual register corresponding to the specified result
267/// of the specified node.
268Register InstrEmitter::getVR(SDValue Op,
269 DenseMap<SDValue, Register> &VRBaseMap) {
270 if (Op.isMachineOpcode() &&
271 Op.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF) {
272 // Add an IMPLICIT_DEF instruction before every use.
273 // IMPLICIT_DEF can produce any type of result so its MCInstrDesc
274 // does not include operand register class info.
275 const TargetRegisterClass *RC = TLI->getRegClassFor(
276 VT: Op.getSimpleValueType(), isDivergent: Op.getNode()->isDivergent());
277 Register VReg = MRI->createVirtualRegister(RegClass: RC);
278 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Op.getDebugLoc(),
279 MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: VReg);
280 return VReg;
281 }
282
283 DenseMap<SDValue, Register>::iterator I = VRBaseMap.find(Val: Op);
284 assert(I != VRBaseMap.end() && "Node emitted out of order - late");
285 return I->second;
286}
287
288static bool isConvergenceCtrlMachineOp(SDValue Op) {
289 if (Op->isMachineOpcode()) {
290 switch (Op->getMachineOpcode()) {
291 case TargetOpcode::CONVERGENCECTRL_ANCHOR:
292 case TargetOpcode::CONVERGENCECTRL_ENTRY:
293 case TargetOpcode::CONVERGENCECTRL_LOOP:
294 case TargetOpcode::CONVERGENCECTRL_GLUE:
295 return true;
296 }
297 return false;
298 }
299
300 // We can reach here when CopyFromReg is encountered. But rather than making a
301 // special case for that, we just make sure we don't reach here in some
302 // surprising way.
303 switch (Op->getOpcode()) {
304 case ISD::CONVERGENCECTRL_ANCHOR:
305 case ISD::CONVERGENCECTRL_ENTRY:
306 case ISD::CONVERGENCECTRL_LOOP:
307 case ISD::CONVERGENCECTRL_GLUE:
308 llvm_unreachable("Convergence control should have been selected by now.");
309 }
310 return false;
311}
312
313/// AddRegisterOperand - Add the specified register as an operand to the
314/// specified machine instr. Insert register copies if the register is
315/// not in the required register class.
316void
317InstrEmitter::AddRegisterOperand(MachineInstrBuilder &MIB,
318 SDValue Op,
319 unsigned IIOpNum,
320 const MCInstrDesc *II,
321 DenseMap<SDValue, Register> &VRBaseMap,
322 bool IsDebug, bool IsClone, bool IsCloned) {
323 assert(Op.getValueType() != MVT::Other &&
324 Op.getValueType() != MVT::Glue &&
325 "Chain and glue operands should occur at end of operand list!");
326 // Get/emit the operand.
327 Register VReg = getVR(Op, VRBaseMap);
328
329 const MCInstrDesc &MCID = MIB->getDesc();
330 bool isOptDef = IIOpNum < MCID.getNumOperands() &&
331 MCID.operands()[IIOpNum].isOptionalDef();
332
333 // If the instruction requires a register in a different class, create
334 // a new virtual register and copy the value into it, but first attempt to
335 // shrink VReg's register class within reason. For example, if VReg == GR32
336 // and II requires a GR32_NOSP, just constrain VReg to GR32_NOSP.
337 if (II) {
338 const TargetRegisterClass *OpRC = nullptr;
339 if (IIOpNum < II->getNumOperands())
340 OpRC = TII->getRegClass(MCID: *II, OpNum: IIOpNum, TRI, MF: *MF);
341
342 if (OpRC) {
343 unsigned MinNumRegs = MinRCSize;
344 // Don't apply any RC size limit for IMPLICIT_DEF. Each use has a unique
345 // virtual register.
346 if (Op.isMachineOpcode() &&
347 Op.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF)
348 MinNumRegs = 0;
349
350 const TargetRegisterClass *ConstrainedRC
351 = MRI->constrainRegClass(Reg: VReg, RC: OpRC, MinNumRegs);
352 if (!ConstrainedRC) {
353 OpRC = TRI->getAllocatableClass(RC: OpRC);
354 assert(OpRC && "Constraints cannot be fulfilled for allocation");
355 Register NewVReg = MRI->createVirtualRegister(RegClass: OpRC);
356 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Op.getNode()->getDebugLoc(),
357 MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: NewVReg).addReg(RegNo: VReg);
358 VReg = NewVReg;
359 } else {
360 assert(ConstrainedRC->isAllocatable() &&
361 "Constraining an allocatable VReg produced an unallocatable class?");
362 }
363 }
364 }
365
366 // If this value has only one use, that use is a kill. This is a
367 // conservative approximation. InstrEmitter does trivial coalescing
368 // with CopyFromReg nodes, so don't emit kill flags for them.
369 // Avoid kill flags on Schedule cloned nodes, since there will be
370 // multiple uses.
371 // Tied operands are never killed, so we need to check that. And that
372 // means we need to determine the index of the operand.
373 // Don't kill convergence control tokens. Initially they are only used in glue
374 // nodes, and the InstrEmitter later adds implicit uses on the users of the
375 // glue node. This can sometimes make it seem like there is only one use,
376 // which is the glue node itself.
377 bool isKill = Op.hasOneUse() && !isConvergenceCtrlMachineOp(Op) &&
378 Op.getNode()->getOpcode() != ISD::CopyFromReg && !IsDebug &&
379 !(IsClone || IsCloned);
380 if (isKill) {
381 unsigned Idx = MIB->getNumOperands();
382 while (Idx > 0 &&
383 MIB->getOperand(i: Idx-1).isReg() &&
384 MIB->getOperand(i: Idx-1).isImplicit())
385 --Idx;
386 bool isTied = MCID.getOperandConstraint(OpNum: Idx, Constraint: MCOI::TIED_TO) != -1;
387 if (isTied)
388 isKill = false;
389 }
390
391 MIB.addReg(RegNo: VReg, flags: getDefRegState(B: isOptDef) | getKillRegState(B: isKill) |
392 getDebugRegState(B: IsDebug));
393}
394
395/// AddOperand - Add the specified operand to the specified machine instr. II
396/// specifies the instruction information for the node, and IIOpNum is the
397/// operand number (in the II) that we are adding.
398void InstrEmitter::AddOperand(MachineInstrBuilder &MIB,
399 SDValue Op,
400 unsigned IIOpNum,
401 const MCInstrDesc *II,
402 DenseMap<SDValue, Register> &VRBaseMap,
403 bool IsDebug, bool IsClone, bool IsCloned) {
404 if (Op.isMachineOpcode()) {
405 AddRegisterOperand(MIB, Op, IIOpNum, II, VRBaseMap,
406 IsDebug, IsClone, IsCloned);
407 } else if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
408 MIB.addImm(Val: C->getSExtValue());
409 } else if (ConstantFPSDNode *F = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
410 MIB.addFPImm(Val: F->getConstantFPValue());
411 } else if (RegisterSDNode *R = dyn_cast<RegisterSDNode>(Val&: Op)) {
412 Register VReg = R->getReg();
413 MVT OpVT = Op.getSimpleValueType();
414 const TargetRegisterClass *IIRC =
415 II ? TRI->getAllocatableClass(RC: TII->getRegClass(MCID: *II, OpNum: IIOpNum, TRI, MF: *MF))
416 : nullptr;
417 const TargetRegisterClass *OpRC =
418 TLI->isTypeLegal(VT: OpVT)
419 ? TLI->getRegClassFor(VT: OpVT,
420 isDivergent: Op.getNode()->isDivergent() ||
421 (IIRC && TRI->isDivergentRegClass(RC: IIRC)))
422 : nullptr;
423
424 if (OpRC && IIRC && OpRC != IIRC && VReg.isVirtual()) {
425 Register NewVReg = MRI->createVirtualRegister(RegClass: IIRC);
426 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Op.getNode()->getDebugLoc(),
427 MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: NewVReg).addReg(RegNo: VReg);
428 VReg = NewVReg;
429 }
430 // Turn additional physreg operands into implicit uses on non-variadic
431 // instructions. This is used by call and return instructions passing
432 // arguments in registers.
433 bool Imp = II && (IIOpNum >= II->getNumOperands() && !II->isVariadic());
434 MIB.addReg(RegNo: VReg, flags: getImplRegState(B: Imp));
435 } else if (RegisterMaskSDNode *RM = dyn_cast<RegisterMaskSDNode>(Val&: Op)) {
436 MIB.addRegMask(Mask: RM->getRegMask());
437 } else if (GlobalAddressSDNode *TGA = dyn_cast<GlobalAddressSDNode>(Val&: Op)) {
438 MIB.addGlobalAddress(GV: TGA->getGlobal(), Offset: TGA->getOffset(),
439 TargetFlags: TGA->getTargetFlags());
440 } else if (BasicBlockSDNode *BBNode = dyn_cast<BasicBlockSDNode>(Val&: Op)) {
441 MIB.addMBB(MBB: BBNode->getBasicBlock());
442 } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Op)) {
443 MIB.addFrameIndex(Idx: FI->getIndex());
444 } else if (JumpTableSDNode *JT = dyn_cast<JumpTableSDNode>(Val&: Op)) {
445 MIB.addJumpTableIndex(Idx: JT->getIndex(), TargetFlags: JT->getTargetFlags());
446 } else if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(Val&: Op)) {
447 int Offset = CP->getOffset();
448 Align Alignment = CP->getAlign();
449
450 unsigned Idx;
451 MachineConstantPool *MCP = MF->getConstantPool();
452 if (CP->isMachineConstantPoolEntry())
453 Idx = MCP->getConstantPoolIndex(V: CP->getMachineCPVal(), Alignment);
454 else
455 Idx = MCP->getConstantPoolIndex(C: CP->getConstVal(), Alignment);
456 MIB.addConstantPoolIndex(Idx, Offset, TargetFlags: CP->getTargetFlags());
457 } else if (ExternalSymbolSDNode *ES = dyn_cast<ExternalSymbolSDNode>(Val&: Op)) {
458 MIB.addExternalSymbol(FnName: ES->getSymbol(), TargetFlags: ES->getTargetFlags());
459 } else if (auto *SymNode = dyn_cast<MCSymbolSDNode>(Val&: Op)) {
460 MIB.addSym(Sym: SymNode->getMCSymbol());
461 } else if (BlockAddressSDNode *BA = dyn_cast<BlockAddressSDNode>(Val&: Op)) {
462 MIB.addBlockAddress(BA: BA->getBlockAddress(),
463 Offset: BA->getOffset(),
464 TargetFlags: BA->getTargetFlags());
465 } else if (TargetIndexSDNode *TI = dyn_cast<TargetIndexSDNode>(Val&: Op)) {
466 MIB.addTargetIndex(Idx: TI->getIndex(), Offset: TI->getOffset(), TargetFlags: TI->getTargetFlags());
467 } else {
468 assert(Op.getValueType() != MVT::Other &&
469 Op.getValueType() != MVT::Glue &&
470 "Chain and glue operands should occur at end of operand list!");
471 AddRegisterOperand(MIB, Op, IIOpNum, II, VRBaseMap,
472 IsDebug, IsClone, IsCloned);
473 }
474}
475
476Register InstrEmitter::ConstrainForSubReg(Register VReg, unsigned SubIdx,
477 MVT VT, bool isDivergent, const DebugLoc &DL) {
478 const TargetRegisterClass *VRC = MRI->getRegClass(Reg: VReg);
479 const TargetRegisterClass *RC = TRI->getSubClassWithSubReg(RC: VRC, Idx: SubIdx);
480
481 // RC is a sub-class of VRC that supports SubIdx. Try to constrain VReg
482 // within reason.
483 if (RC && RC != VRC)
484 RC = MRI->constrainRegClass(Reg: VReg, RC, MinNumRegs: MinRCSize);
485
486 // VReg has been adjusted. It can be used with SubIdx operands now.
487 if (RC)
488 return VReg;
489
490 // VReg couldn't be reasonably constrained. Emit a COPY to a new virtual
491 // register instead.
492 RC = TRI->getSubClassWithSubReg(RC: TLI->getRegClassFor(VT, isDivergent), Idx: SubIdx);
493 assert(RC && "No legal register class for VT supports that SubIdx");
494 Register NewReg = MRI->createVirtualRegister(RegClass: RC);
495 BuildMI(BB&: *MBB, I: InsertPos, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: NewReg)
496 .addReg(RegNo: VReg);
497 return NewReg;
498}
499
500/// EmitSubregNode - Generate machine code for subreg nodes.
501///
502void InstrEmitter::EmitSubregNode(SDNode *Node,
503 DenseMap<SDValue, Register> &VRBaseMap,
504 bool IsClone, bool IsCloned) {
505 Register VRBase;
506 unsigned Opc = Node->getMachineOpcode();
507
508 // If the node is only used by a CopyToReg and the dest reg is a vreg, use
509 // the CopyToReg'd destination register instead of creating a new vreg.
510 for (SDNode *User : Node->uses()) {
511 if (User->getOpcode() == ISD::CopyToReg &&
512 User->getOperand(Num: 2).getNode() == Node) {
513 Register DestReg = cast<RegisterSDNode>(Val: User->getOperand(Num: 1))->getReg();
514 if (DestReg.isVirtual()) {
515 VRBase = DestReg;
516 break;
517 }
518 }
519 }
520
521 if (Opc == TargetOpcode::EXTRACT_SUBREG) {
522 // EXTRACT_SUBREG is lowered as %dst = COPY %src:sub. There are no
523 // constraints on the %dst register, COPY can target all legal register
524 // classes.
525 unsigned SubIdx = Node->getConstantOperandVal(Num: 1);
526 const TargetRegisterClass *TRC =
527 TLI->getRegClassFor(VT: Node->getSimpleValueType(ResNo: 0), isDivergent: Node->isDivergent());
528
529 Register Reg;
530 MachineInstr *DefMI;
531 RegisterSDNode *R = dyn_cast<RegisterSDNode>(Val: Node->getOperand(Num: 0));
532 if (R && R->getReg().isPhysical()) {
533 Reg = R->getReg();
534 DefMI = nullptr;
535 } else {
536 Reg = R ? R->getReg() : getVR(Op: Node->getOperand(Num: 0), VRBaseMap);
537 DefMI = MRI->getVRegDef(Reg);
538 }
539
540 Register SrcReg, DstReg;
541 unsigned DefSubIdx;
542 if (DefMI &&
543 TII->isCoalescableExtInstr(MI: *DefMI, SrcReg, DstReg, SubIdx&: DefSubIdx) &&
544 SubIdx == DefSubIdx &&
545 TRC == MRI->getRegClass(Reg: SrcReg)) {
546 // Optimize these:
547 // r1025 = s/zext r1024, 4
548 // r1026 = extract_subreg r1025, 4
549 // to a copy
550 // r1026 = copy r1024
551 VRBase = MRI->createVirtualRegister(RegClass: TRC);
552 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Node->getDebugLoc(),
553 MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: VRBase).addReg(RegNo: SrcReg);
554 MRI->clearKillFlags(Reg: SrcReg);
555 } else {
556 // Reg may not support a SubIdx sub-register, and we may need to
557 // constrain its register class or issue a COPY to a compatible register
558 // class.
559 if (Reg.isVirtual())
560 Reg = ConstrainForSubReg(VReg: Reg, SubIdx,
561 VT: Node->getOperand(Num: 0).getSimpleValueType(),
562 isDivergent: Node->isDivergent(), DL: Node->getDebugLoc());
563 // Create the destreg if it is missing.
564 if (!VRBase)
565 VRBase = MRI->createVirtualRegister(RegClass: TRC);
566
567 // Create the extract_subreg machine instruction.
568 MachineInstrBuilder CopyMI =
569 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Node->getDebugLoc(),
570 MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: VRBase);
571 if (Reg.isVirtual())
572 CopyMI.addReg(RegNo: Reg, flags: 0, SubReg: SubIdx);
573 else
574 CopyMI.addReg(RegNo: TRI->getSubReg(Reg, Idx: SubIdx));
575 }
576 } else if (Opc == TargetOpcode::INSERT_SUBREG ||
577 Opc == TargetOpcode::SUBREG_TO_REG) {
578 SDValue N0 = Node->getOperand(Num: 0);
579 SDValue N1 = Node->getOperand(Num: 1);
580 SDValue N2 = Node->getOperand(Num: 2);
581 unsigned SubIdx = N2->getAsZExtVal();
582
583 // Figure out the register class to create for the destreg. It should be
584 // the largest legal register class supporting SubIdx sub-registers.
585 // RegisterCoalescer will constrain it further if it decides to eliminate
586 // the INSERT_SUBREG instruction.
587 //
588 // %dst = INSERT_SUBREG %src, %sub, SubIdx
589 //
590 // is lowered by TwoAddressInstructionPass to:
591 //
592 // %dst = COPY %src
593 // %dst:SubIdx = COPY %sub
594 //
595 // There is no constraint on the %src register class.
596 //
597 const TargetRegisterClass *SRC =
598 TLI->getRegClassFor(VT: Node->getSimpleValueType(ResNo: 0), isDivergent: Node->isDivergent());
599 SRC = TRI->getSubClassWithSubReg(RC: SRC, Idx: SubIdx);
600 assert(SRC && "No register class supports VT and SubIdx for INSERT_SUBREG");
601
602 if (VRBase == 0 || !SRC->hasSubClassEq(RC: MRI->getRegClass(Reg: VRBase)))
603 VRBase = MRI->createVirtualRegister(RegClass: SRC);
604
605 // Create the insert_subreg or subreg_to_reg machine instruction.
606 MachineInstrBuilder MIB =
607 BuildMI(MF&: *MF, MIMD: Node->getDebugLoc(), MCID: TII->get(Opcode: Opc), DestReg: VRBase);
608
609 // If creating a subreg_to_reg, then the first input operand
610 // is an implicit value immediate, otherwise it's a register
611 if (Opc == TargetOpcode::SUBREG_TO_REG) {
612 const ConstantSDNode *SD = cast<ConstantSDNode>(Val&: N0);
613 MIB.addImm(Val: SD->getZExtValue());
614 } else
615 AddOperand(MIB, Op: N0, IIOpNum: 0, II: nullptr, VRBaseMap, /*IsDebug=*/false,
616 IsClone, IsCloned);
617 // Add the subregister being inserted
618 AddOperand(MIB, Op: N1, IIOpNum: 0, II: nullptr, VRBaseMap, /*IsDebug=*/false,
619 IsClone, IsCloned);
620 MIB.addImm(Val: SubIdx);
621 MBB->insert(I: InsertPos, MI: MIB);
622 } else
623 llvm_unreachable("Node is not insert_subreg, extract_subreg, or subreg_to_reg");
624
625 SDValue Op(Node, 0);
626 bool isNew = VRBaseMap.insert(KV: std::make_pair(x&: Op, y&: VRBase)).second;
627 (void)isNew; // Silence compiler warning.
628 assert(isNew && "Node emitted out of order - early");
629}
630
631/// EmitCopyToRegClassNode - Generate machine code for COPY_TO_REGCLASS nodes.
632/// COPY_TO_REGCLASS is just a normal copy, except that the destination
633/// register is constrained to be in a particular register class.
634///
635void
636InstrEmitter::EmitCopyToRegClassNode(SDNode *Node,
637 DenseMap<SDValue, Register> &VRBaseMap) {
638 unsigned VReg = getVR(Op: Node->getOperand(Num: 0), VRBaseMap);
639
640 // Create the new VReg in the destination class and emit a copy.
641 unsigned DstRCIdx = Node->getConstantOperandVal(Num: 1);
642 const TargetRegisterClass *DstRC =
643 TRI->getAllocatableClass(RC: TRI->getRegClass(i: DstRCIdx));
644 Register NewVReg = MRI->createVirtualRegister(RegClass: DstRC);
645 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Node->getDebugLoc(), MCID: TII->get(Opcode: TargetOpcode::COPY),
646 DestReg: NewVReg).addReg(RegNo: VReg);
647
648 SDValue Op(Node, 0);
649 bool isNew = VRBaseMap.insert(KV: std::make_pair(x&: Op, y&: NewVReg)).second;
650 (void)isNew; // Silence compiler warning.
651 assert(isNew && "Node emitted out of order - early");
652}
653
654/// EmitRegSequence - Generate machine code for REG_SEQUENCE nodes.
655///
656void InstrEmitter::EmitRegSequence(SDNode *Node,
657 DenseMap<SDValue, Register> &VRBaseMap,
658 bool IsClone, bool IsCloned) {
659 unsigned DstRCIdx = Node->getConstantOperandVal(Num: 0);
660 const TargetRegisterClass *RC = TRI->getRegClass(i: DstRCIdx);
661 Register NewVReg = MRI->createVirtualRegister(RegClass: TRI->getAllocatableClass(RC));
662 const MCInstrDesc &II = TII->get(Opcode: TargetOpcode::REG_SEQUENCE);
663 MachineInstrBuilder MIB = BuildMI(MF&: *MF, MIMD: Node->getDebugLoc(), MCID: II, DestReg: NewVReg);
664 unsigned NumOps = Node->getNumOperands();
665 // If the input pattern has a chain, then the root of the corresponding
666 // output pattern will get a chain as well. This can happen to be a
667 // REG_SEQUENCE (which is not "guarded" by countOperands/CountResults).
668 if (NumOps && Node->getOperand(Num: NumOps-1).getValueType() == MVT::Other)
669 --NumOps; // Ignore chain if it exists.
670
671 assert((NumOps & 1) == 1 &&
672 "REG_SEQUENCE must have an odd number of operands!");
673 for (unsigned i = 1; i != NumOps; ++i) {
674 SDValue Op = Node->getOperand(Num: i);
675 if ((i & 1) == 0) {
676 RegisterSDNode *R = dyn_cast<RegisterSDNode>(Val: Node->getOperand(Num: i-1));
677 // Skip physical registers as they don't have a vreg to get and we'll
678 // insert copies for them in TwoAddressInstructionPass anyway.
679 if (!R || !R->getReg().isPhysical()) {
680 unsigned SubIdx = Op->getAsZExtVal();
681 unsigned SubReg = getVR(Op: Node->getOperand(Num: i-1), VRBaseMap);
682 const TargetRegisterClass *TRC = MRI->getRegClass(Reg: SubReg);
683 const TargetRegisterClass *SRC =
684 TRI->getMatchingSuperRegClass(A: RC, B: TRC, Idx: SubIdx);
685 if (SRC && SRC != RC) {
686 MRI->setRegClass(Reg: NewVReg, RC: SRC);
687 RC = SRC;
688 }
689 }
690 }
691 AddOperand(MIB, Op, IIOpNum: i+1, II: &II, VRBaseMap, /*IsDebug=*/false,
692 IsClone, IsCloned);
693 }
694
695 MBB->insert(I: InsertPos, MI: MIB);
696 SDValue Op(Node, 0);
697 bool isNew = VRBaseMap.insert(KV: std::make_pair(x&: Op, y&: NewVReg)).second;
698 (void)isNew; // Silence compiler warning.
699 assert(isNew && "Node emitted out of order - early");
700}
701
702/// EmitDbgValue - Generate machine instruction for a dbg_value node.
703///
704MachineInstr *
705InstrEmitter::EmitDbgValue(SDDbgValue *SD,
706 DenseMap<SDValue, Register> &VRBaseMap) {
707 DebugLoc DL = SD->getDebugLoc();
708 assert(cast<DILocalVariable>(SD->getVariable())
709 ->isValidLocationForIntrinsic(DL) &&
710 "Expected inlined-at fields to agree");
711
712 SD->setIsEmitted();
713
714 assert(!SD->getLocationOps().empty() &&
715 "dbg_value with no location operands?");
716
717 if (SD->isInvalidated())
718 return EmitDbgNoLocation(SD);
719
720 // Attempt to produce a DBG_INSTR_REF if we've been asked to.
721 if (EmitDebugInstrRefs)
722 if (auto *InstrRef = EmitDbgInstrRef(SD, VRBaseMap))
723 return InstrRef;
724
725 // Emit variadic dbg_value nodes as DBG_VALUE_LIST if they have not been
726 // emitted as instruction references.
727 if (SD->isVariadic())
728 return EmitDbgValueList(SD, VRBaseMap);
729
730 // Emit single-location dbg_value nodes as DBG_VALUE if they have not been
731 // emitted as instruction references.
732 return EmitDbgValueFromSingleOp(SD, VRBaseMap);
733}
734
735MachineOperand GetMOForConstDbgOp(const SDDbgOperand &Op) {
736 const Value *V = Op.getConst();
737 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: V)) {
738 if (CI->getBitWidth() > 64)
739 return MachineOperand::CreateCImm(CI);
740 return MachineOperand::CreateImm(Val: CI->getSExtValue());
741 }
742 if (const ConstantFP *CF = dyn_cast<ConstantFP>(Val: V))
743 return MachineOperand::CreateFPImm(CFP: CF);
744 // Note: This assumes that all nullptr constants are zero-valued.
745 if (isa<ConstantPointerNull>(Val: V))
746 return MachineOperand::CreateImm(Val: 0);
747 // Undef or unhandled value type, so return an undef operand.
748 return MachineOperand::CreateReg(
749 /* Reg */ 0U, /* isDef */ false, /* isImp */ false,
750 /* isKill */ false, /* isDead */ false,
751 /* isUndef */ false, /* isEarlyClobber */ false,
752 /* SubReg */ 0, /* isDebug */ true);
753}
754
755void InstrEmitter::AddDbgValueLocationOps(
756 MachineInstrBuilder &MIB, const MCInstrDesc &DbgValDesc,
757 ArrayRef<SDDbgOperand> LocationOps,
758 DenseMap<SDValue, Register> &VRBaseMap) {
759 for (const SDDbgOperand &Op : LocationOps) {
760 switch (Op.getKind()) {
761 case SDDbgOperand::FRAMEIX:
762 MIB.addFrameIndex(Idx: Op.getFrameIx());
763 break;
764 case SDDbgOperand::VREG:
765 MIB.addReg(RegNo: Op.getVReg());
766 break;
767 case SDDbgOperand::SDNODE: {
768 SDValue V = SDValue(Op.getSDNode(), Op.getResNo());
769 // It's possible we replaced this SDNode with other(s) and therefore
770 // didn't generate code for it. It's better to catch these cases where
771 // they happen and transfer the debug info, but trying to guarantee that
772 // in all cases would be very fragile; this is a safeguard for any
773 // that were missed.
774 if (VRBaseMap.count(Val: V) == 0)
775 MIB.addReg(RegNo: 0U); // undef
776 else
777 AddOperand(MIB, Op: V, IIOpNum: (*MIB).getNumOperands(), II: &DbgValDesc, VRBaseMap,
778 /*IsDebug=*/true, /*IsClone=*/false, /*IsCloned=*/false);
779 } break;
780 case SDDbgOperand::CONST:
781 MIB.add(MO: GetMOForConstDbgOp(Op));
782 break;
783 }
784 }
785}
786
787MachineInstr *
788InstrEmitter::EmitDbgInstrRef(SDDbgValue *SD,
789 DenseMap<SDValue, Register> &VRBaseMap) {
790 MDNode *Var = SD->getVariable();
791 const DIExpression *Expr = (DIExpression *)SD->getExpression();
792 DebugLoc DL = SD->getDebugLoc();
793 const MCInstrDesc &RefII = TII->get(Opcode: TargetOpcode::DBG_INSTR_REF);
794
795 // Returns true if the given operand is not a legal debug operand for a
796 // DBG_INSTR_REF.
797 auto IsInvalidOp = [](SDDbgOperand DbgOp) {
798 return DbgOp.getKind() == SDDbgOperand::FRAMEIX;
799 };
800 // Returns true if the given operand is not itself an instruction reference
801 // but is a legal debug operand for a DBG_INSTR_REF.
802 auto IsNonInstrRefOp = [](SDDbgOperand DbgOp) {
803 return DbgOp.getKind() == SDDbgOperand::CONST;
804 };
805
806 // If this variable location does not depend on any instructions or contains
807 // any stack locations, produce it as a standard debug value instead.
808 if (any_of(Range: SD->getLocationOps(), P: IsInvalidOp) ||
809 all_of(Range: SD->getLocationOps(), P: IsNonInstrRefOp)) {
810 if (SD->isVariadic())
811 return EmitDbgValueList(SD, VRBaseMap);
812 return EmitDbgValueFromSingleOp(SD, VRBaseMap);
813 }
814
815 // Immediately fold any indirectness from the LLVM-IR intrinsic into the
816 // expression:
817 if (SD->isIndirect())
818 Expr = DIExpression::append(Expr, Ops: dwarf::DW_OP_deref);
819 // If this is not already a variadic expression, it must be modified to become
820 // one.
821 if (!SD->isVariadic())
822 Expr = DIExpression::convertToVariadicExpression(Expr);
823
824 SmallVector<MachineOperand> MOs;
825
826 // It may not be immediately possible to identify the MachineInstr that
827 // defines a VReg, it can depend for example on the order blocks are
828 // emitted in. When this happens, or when further analysis is needed later,
829 // produce an instruction like this:
830 //
831 // DBG_INSTR_REF !123, !456, %0:gr64
832 //
833 // i.e., point the instruction at the vreg, and patch it up later in
834 // MachineFunction::finalizeDebugInstrRefs.
835 auto AddVRegOp = [&](unsigned VReg) {
836 MOs.push_back(Elt: MachineOperand::CreateReg(
837 /* Reg */ VReg, /* isDef */ false, /* isImp */ false,
838 /* isKill */ false, /* isDead */ false,
839 /* isUndef */ false, /* isEarlyClobber */ false,
840 /* SubReg */ 0, /* isDebug */ true));
841 };
842 unsigned OpCount = SD->getLocationOps().size();
843 for (unsigned OpIdx = 0; OpIdx < OpCount; ++OpIdx) {
844 SDDbgOperand DbgOperand = SD->getLocationOps()[OpIdx];
845
846 // Try to find both the defined register and the instruction defining it.
847 MachineInstr *DefMI = nullptr;
848 unsigned VReg;
849
850 if (DbgOperand.getKind() == SDDbgOperand::VREG) {
851 VReg = DbgOperand.getVReg();
852
853 // No definition means that block hasn't been emitted yet. Leave a vreg
854 // reference to be fixed later.
855 if (!MRI->hasOneDef(RegNo: VReg)) {
856 AddVRegOp(VReg);
857 continue;
858 }
859
860 DefMI = &*MRI->def_instr_begin(RegNo: VReg);
861 } else if (DbgOperand.getKind() == SDDbgOperand::SDNODE) {
862 // Look up the corresponding VReg for the given SDNode, if any.
863 SDNode *Node = DbgOperand.getSDNode();
864 SDValue Op = SDValue(Node, DbgOperand.getResNo());
865 DenseMap<SDValue, Register>::iterator I = VRBaseMap.find(Val: Op);
866 // No VReg -> produce a DBG_VALUE $noreg instead.
867 if (I == VRBaseMap.end())
868 break;
869
870 // Try to pick out a defining instruction at this point.
871 VReg = getVR(Op, VRBaseMap);
872
873 // Again, if there's no instruction defining the VReg right now, fix it up
874 // later.
875 if (!MRI->hasOneDef(RegNo: VReg)) {
876 AddVRegOp(VReg);
877 continue;
878 }
879
880 DefMI = &*MRI->def_instr_begin(RegNo: VReg);
881 } else {
882 assert(DbgOperand.getKind() == SDDbgOperand::CONST);
883 MOs.push_back(Elt: GetMOForConstDbgOp(Op: DbgOperand));
884 continue;
885 }
886
887 // Avoid copy like instructions: they don't define values, only move them.
888 // Leave a virtual-register reference until it can be fixed up later, to
889 // find the underlying value definition.
890 if (DefMI->isCopyLike() || TII->isCopyInstr(MI: *DefMI)) {
891 AddVRegOp(VReg);
892 continue;
893 }
894
895 // Find the operand number which defines the specified VReg.
896 unsigned OperandIdx = 0;
897 for (const auto &MO : DefMI->operands()) {
898 if (MO.isReg() && MO.isDef() && MO.getReg() == VReg)
899 break;
900 ++OperandIdx;
901 }
902 assert(OperandIdx < DefMI->getNumOperands());
903
904 // Make the DBG_INSTR_REF refer to that instruction, and that operand.
905 unsigned InstrNum = DefMI->getDebugInstrNum();
906 MOs.push_back(Elt: MachineOperand::CreateDbgInstrRef(InstrIdx: InstrNum, OpIdx: OperandIdx));
907 }
908
909 // If we haven't created a valid MachineOperand for every DbgOp, abort and
910 // produce an undef DBG_VALUE.
911 if (MOs.size() != OpCount)
912 return EmitDbgNoLocation(SD);
913
914 return BuildMI(MF&: *MF, DL, MCID: RefII, IsIndirect: false, MOs, Variable: Var, Expr);
915}
916
917MachineInstr *InstrEmitter::EmitDbgNoLocation(SDDbgValue *SD) {
918 // An invalidated SDNode must generate an undef DBG_VALUE: although the
919 // original value is no longer computed, earlier DBG_VALUEs live ranges
920 // must not leak into later code.
921 DIVariable *Var = SD->getVariable();
922 const DIExpression *Expr =
923 DIExpression::convertToUndefExpression(Expr: SD->getExpression());
924 DebugLoc DL = SD->getDebugLoc();
925 const MCInstrDesc &Desc = TII->get(Opcode: TargetOpcode::DBG_VALUE);
926 return BuildMI(MF&: *MF, DL, MCID: Desc, IsIndirect: false, Reg: 0U, Variable: Var, Expr);
927}
928
929MachineInstr *
930InstrEmitter::EmitDbgValueList(SDDbgValue *SD,
931 DenseMap<SDValue, Register> &VRBaseMap) {
932 MDNode *Var = SD->getVariable();
933 DIExpression *Expr = SD->getExpression();
934 DebugLoc DL = SD->getDebugLoc();
935 // DBG_VALUE_LIST := "DBG_VALUE_LIST" var, expression, loc (, loc)*
936 const MCInstrDesc &DbgValDesc = TII->get(Opcode: TargetOpcode::DBG_VALUE_LIST);
937 // Build the DBG_VALUE_LIST instruction base.
938 auto MIB = BuildMI(MF&: *MF, MIMD: DL, MCID: DbgValDesc);
939 MIB.addMetadata(MD: Var);
940 MIB.addMetadata(MD: Expr);
941 AddDbgValueLocationOps(MIB, DbgValDesc, LocationOps: SD->getLocationOps(), VRBaseMap);
942 return &*MIB;
943}
944
945MachineInstr *
946InstrEmitter::EmitDbgValueFromSingleOp(SDDbgValue *SD,
947 DenseMap<SDValue, Register> &VRBaseMap) {
948 MDNode *Var = SD->getVariable();
949 DIExpression *Expr = SD->getExpression();
950 DebugLoc DL = SD->getDebugLoc();
951 const MCInstrDesc &II = TII->get(Opcode: TargetOpcode::DBG_VALUE);
952
953 assert(SD->getLocationOps().size() == 1 &&
954 "Non variadic dbg_value should have only one location op");
955
956 // See about constant-folding the expression.
957 // Copy the location operand in case we replace it.
958 SmallVector<SDDbgOperand, 1> LocationOps(1, SD->getLocationOps()[0]);
959 if (Expr && LocationOps[0].getKind() == SDDbgOperand::CONST) {
960 const Value *V = LocationOps[0].getConst();
961 if (auto *C = dyn_cast<ConstantInt>(Val: V)) {
962 std::tie(args&: Expr, args&: C) = Expr->constantFold(CI: C);
963 LocationOps[0] = SDDbgOperand::fromConst(Const: C);
964 }
965 }
966
967 // Emit non-variadic dbg_value nodes as DBG_VALUE.
968 // DBG_VALUE := "DBG_VALUE" loc, isIndirect, var, expr
969 auto MIB = BuildMI(MF&: *MF, MIMD: DL, MCID: II);
970 AddDbgValueLocationOps(MIB, DbgValDesc: II, LocationOps, VRBaseMap);
971
972 if (SD->isIndirect())
973 MIB.addImm(Val: 0U);
974 else
975 MIB.addReg(RegNo: 0U);
976
977 return MIB.addMetadata(MD: Var).addMetadata(MD: Expr);
978}
979
980MachineInstr *
981InstrEmitter::EmitDbgLabel(SDDbgLabel *SD) {
982 MDNode *Label = SD->getLabel();
983 DebugLoc DL = SD->getDebugLoc();
984 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) &&
985 "Expected inlined-at fields to agree");
986
987 const MCInstrDesc &II = TII->get(Opcode: TargetOpcode::DBG_LABEL);
988 MachineInstrBuilder MIB = BuildMI(MF&: *MF, MIMD: DL, MCID: II);
989 MIB.addMetadata(MD: Label);
990
991 return &*MIB;
992}
993
994/// EmitMachineNode - Generate machine code for a target-specific node and
995/// needed dependencies.
996///
997void InstrEmitter::
998EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
999 DenseMap<SDValue, Register> &VRBaseMap) {
1000 unsigned Opc = Node->getMachineOpcode();
1001
1002 // Handle subreg insert/extract specially
1003 if (Opc == TargetOpcode::EXTRACT_SUBREG ||
1004 Opc == TargetOpcode::INSERT_SUBREG ||
1005 Opc == TargetOpcode::SUBREG_TO_REG) {
1006 EmitSubregNode(Node, VRBaseMap, IsClone, IsCloned);
1007 return;
1008 }
1009
1010 // Handle COPY_TO_REGCLASS specially.
1011 if (Opc == TargetOpcode::COPY_TO_REGCLASS) {
1012 EmitCopyToRegClassNode(Node, VRBaseMap);
1013 return;
1014 }
1015
1016 // Handle REG_SEQUENCE specially.
1017 if (Opc == TargetOpcode::REG_SEQUENCE) {
1018 EmitRegSequence(Node, VRBaseMap, IsClone, IsCloned);
1019 return;
1020 }
1021
1022 if (Opc == TargetOpcode::IMPLICIT_DEF)
1023 // We want a unique VR for each IMPLICIT_DEF use.
1024 return;
1025
1026 const MCInstrDesc &II = TII->get(Opcode: Opc);
1027 unsigned NumResults = CountResults(Node);
1028 unsigned NumDefs = II.getNumDefs();
1029 const MCPhysReg *ScratchRegs = nullptr;
1030
1031 // Handle STACKMAP and PATCHPOINT specially and then use the generic code.
1032 if (Opc == TargetOpcode::STACKMAP || Opc == TargetOpcode::PATCHPOINT) {
1033 // Stackmaps do not have arguments and do not preserve their calling
1034 // convention. However, to simplify runtime support, they clobber the same
1035 // scratch registers as AnyRegCC.
1036 unsigned CC = CallingConv::AnyReg;
1037 if (Opc == TargetOpcode::PATCHPOINT) {
1038 CC = Node->getConstantOperandVal(Num: PatchPointOpers::CCPos);
1039 NumDefs = NumResults;
1040 }
1041 ScratchRegs = TLI->getScratchRegisters(CC: (CallingConv::ID) CC);
1042 } else if (Opc == TargetOpcode::STATEPOINT) {
1043 NumDefs = NumResults;
1044 }
1045
1046 unsigned NumImpUses = 0;
1047 unsigned NodeOperands =
1048 countOperands(Node, NumExpUses: II.getNumOperands() - NumDefs, NumImpUses);
1049 bool HasVRegVariadicDefs = !MF->getTarget().usesPhysRegsForValues() &&
1050 II.isVariadic() && II.variadicOpsAreDefs();
1051 bool HasPhysRegOuts = NumResults > NumDefs && !II.implicit_defs().empty() &&
1052 !HasVRegVariadicDefs;
1053#ifndef NDEBUG
1054 unsigned NumMIOperands = NodeOperands + NumResults;
1055 if (II.isVariadic())
1056 assert(NumMIOperands >= II.getNumOperands() &&
1057 "Too few operands for a variadic node!");
1058 else
1059 assert(NumMIOperands >= II.getNumOperands() &&
1060 NumMIOperands <=
1061 II.getNumOperands() + II.implicit_defs().size() + NumImpUses &&
1062 "#operands for dag node doesn't match .td file!");
1063#endif
1064
1065 // Create the new machine instruction.
1066 MachineInstrBuilder MIB = BuildMI(MF&: *MF, MIMD: Node->getDebugLoc(), MCID: II);
1067
1068 // Add result register values for things that are defined by this
1069 // instruction.
1070 if (NumResults) {
1071 CreateVirtualRegisters(Node, MIB, II, IsClone, IsCloned, VRBaseMap);
1072
1073 // Transfer any IR flags from the SDNode to the MachineInstr
1074 MachineInstr *MI = MIB.getInstr();
1075 const SDNodeFlags Flags = Node->getFlags();
1076 if (Flags.hasNoSignedZeros())
1077 MI->setFlag(MachineInstr::MIFlag::FmNsz);
1078
1079 if (Flags.hasAllowReciprocal())
1080 MI->setFlag(MachineInstr::MIFlag::FmArcp);
1081
1082 if (Flags.hasNoNaNs())
1083 MI->setFlag(MachineInstr::MIFlag::FmNoNans);
1084
1085 if (Flags.hasNoInfs())
1086 MI->setFlag(MachineInstr::MIFlag::FmNoInfs);
1087
1088 if (Flags.hasAllowContract())
1089 MI->setFlag(MachineInstr::MIFlag::FmContract);
1090
1091 if (Flags.hasApproximateFuncs())
1092 MI->setFlag(MachineInstr::MIFlag::FmAfn);
1093
1094 if (Flags.hasAllowReassociation())
1095 MI->setFlag(MachineInstr::MIFlag::FmReassoc);
1096
1097 if (Flags.hasNoUnsignedWrap())
1098 MI->setFlag(MachineInstr::MIFlag::NoUWrap);
1099
1100 if (Flags.hasNoSignedWrap())
1101 MI->setFlag(MachineInstr::MIFlag::NoSWrap);
1102
1103 if (Flags.hasExact())
1104 MI->setFlag(MachineInstr::MIFlag::IsExact);
1105
1106 if (Flags.hasNoFPExcept())
1107 MI->setFlag(MachineInstr::MIFlag::NoFPExcept);
1108
1109 if (Flags.hasUnpredictable())
1110 MI->setFlag(MachineInstr::MIFlag::Unpredictable);
1111 }
1112
1113 // Emit all of the actual operands of this instruction, adding them to the
1114 // instruction as appropriate.
1115 bool HasOptPRefs = NumDefs > NumResults;
1116 assert((!HasOptPRefs || !HasPhysRegOuts) &&
1117 "Unable to cope with optional defs and phys regs defs!");
1118 unsigned NumSkip = HasOptPRefs ? NumDefs - NumResults : 0;
1119 for (unsigned i = NumSkip; i != NodeOperands; ++i)
1120 AddOperand(MIB, Op: Node->getOperand(Num: i), IIOpNum: i-NumSkip+NumDefs, II: &II,
1121 VRBaseMap, /*IsDebug=*/false, IsClone, IsCloned);
1122
1123 // Add scratch registers as implicit def and early clobber
1124 if (ScratchRegs)
1125 for (unsigned i = 0; ScratchRegs[i]; ++i)
1126 MIB.addReg(RegNo: ScratchRegs[i], flags: RegState::ImplicitDefine |
1127 RegState::EarlyClobber);
1128
1129 // Set the memory reference descriptions of this instruction now that it is
1130 // part of the function.
1131 MIB.setMemRefs(cast<MachineSDNode>(Val: Node)->memoperands());
1132
1133 // Set the CFI type.
1134 MIB->setCFIType(MF&: *MF, Type: Node->getCFIType());
1135
1136 // Insert the instruction into position in the block. This needs to
1137 // happen before any custom inserter hook is called so that the
1138 // hook knows where in the block to insert the replacement code.
1139 MBB->insert(I: InsertPos, MI: MIB);
1140
1141 // The MachineInstr may also define physregs instead of virtregs. These
1142 // physreg values can reach other instructions in different ways:
1143 //
1144 // 1. When there is a use of a Node value beyond the explicitly defined
1145 // virtual registers, we emit a CopyFromReg for one of the implicitly
1146 // defined physregs. This only happens when HasPhysRegOuts is true.
1147 //
1148 // 2. A CopyFromReg reading a physreg may be glued to this instruction.
1149 //
1150 // 3. A glued instruction may implicitly use a physreg.
1151 //
1152 // 4. A glued instruction may use a RegisterSDNode operand.
1153 //
1154 // Collect all the used physreg defs, and make sure that any unused physreg
1155 // defs are marked as dead.
1156 SmallVector<Register, 8> UsedRegs;
1157
1158 // Additional results must be physical register defs.
1159 if (HasPhysRegOuts) {
1160 for (unsigned i = NumDefs; i < NumResults; ++i) {
1161 Register Reg = II.implicit_defs()[i - NumDefs];
1162 if (!Node->hasAnyUseOfValue(Value: i))
1163 continue;
1164 // This implicitly defined physreg has a use.
1165 UsedRegs.push_back(Elt: Reg);
1166 EmitCopyFromReg(Node, ResNo: i, IsClone, SrcReg: Reg, VRBaseMap);
1167 }
1168 }
1169
1170 // Scan the glue chain for any used physregs.
1171 if (Node->getValueType(ResNo: Node->getNumValues()-1) == MVT::Glue) {
1172 for (SDNode *F = Node->getGluedUser(); F; F = F->getGluedUser()) {
1173 if (F->getOpcode() == ISD::CopyFromReg) {
1174 UsedRegs.push_back(Elt: cast<RegisterSDNode>(Val: F->getOperand(Num: 1))->getReg());
1175 continue;
1176 } else if (F->getOpcode() == ISD::CopyToReg) {
1177 // Skip CopyToReg nodes that are internal to the glue chain.
1178 continue;
1179 }
1180 // Collect declared implicit uses.
1181 const MCInstrDesc &MCID = TII->get(Opcode: F->getMachineOpcode());
1182 append_range(C&: UsedRegs, R: MCID.implicit_uses());
1183 // In addition to declared implicit uses, we must also check for
1184 // direct RegisterSDNode operands.
1185 for (const SDValue &Op : F->op_values())
1186 if (RegisterSDNode *R = dyn_cast<RegisterSDNode>(Val: Op)) {
1187 Register Reg = R->getReg();
1188 if (Reg.isPhysical())
1189 UsedRegs.push_back(Elt: Reg);
1190 }
1191 }
1192 }
1193
1194 // Add rounding control registers as implicit def for function call.
1195 if (II.isCall() && MF->getFunction().hasFnAttribute(Kind: Attribute::StrictFP)) {
1196 ArrayRef<MCPhysReg> RCRegs = TLI->getRoundingControlRegisters();
1197 for (MCPhysReg Reg : RCRegs)
1198 UsedRegs.push_back(Elt: Reg);
1199 }
1200
1201 // Finally mark unused registers as dead.
1202 if (!UsedRegs.empty() || !II.implicit_defs().empty() || II.hasOptionalDef())
1203 MIB->setPhysRegsDeadExcept(UsedRegs, TRI: *TRI);
1204
1205 // STATEPOINT is too 'dynamic' to have meaningful machine description.
1206 // We have to manually tie operands.
1207 if (Opc == TargetOpcode::STATEPOINT && NumDefs > 0) {
1208 assert(!HasPhysRegOuts && "STATEPOINT mishandled");
1209 MachineInstr *MI = MIB;
1210 unsigned Def = 0;
1211 int First = StatepointOpers(MI).getFirstGCPtrIdx();
1212 assert(First > 0 && "Statepoint has Defs but no GC ptr list");
1213 unsigned Use = (unsigned)First;
1214 while (Def < NumDefs) {
1215 if (MI->getOperand(i: Use).isReg())
1216 MI->tieOperands(DefIdx: Def++, UseIdx: Use);
1217 Use = StackMaps::getNextMetaArgIdx(MI, CurIdx: Use);
1218 }
1219 }
1220
1221 if (SDNode *GluedNode = Node->getGluedNode()) {
1222 // FIXME: Possibly iterate over multiple glue nodes?
1223 if (GluedNode->getOpcode() ==
1224 ~(unsigned)TargetOpcode::CONVERGENCECTRL_GLUE) {
1225 Register VReg = getVR(Op: GluedNode->getOperand(Num: 0), VRBaseMap);
1226 MachineOperand MO = MachineOperand::CreateReg(Reg: VReg, /*isDef=*/false,
1227 /*isImp=*/true);
1228 MIB->addOperand(Op: MO);
1229 }
1230 }
1231
1232 // Run post-isel target hook to adjust this instruction if needed.
1233 if (II.hasPostISelHook())
1234 TLI->AdjustInstrPostInstrSelection(MI&: *MIB, Node);
1235}
1236
1237/// EmitSpecialNode - Generate machine code for a target-independent node and
1238/// needed dependencies.
1239void InstrEmitter::
1240EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned,
1241 DenseMap<SDValue, Register> &VRBaseMap) {
1242 switch (Node->getOpcode()) {
1243 default:
1244#ifndef NDEBUG
1245 Node->dump();
1246#endif
1247 llvm_unreachable("This target-independent node should have been selected!");
1248 case ISD::EntryToken:
1249 case ISD::MERGE_VALUES:
1250 case ISD::TokenFactor: // fall thru
1251 break;
1252 case ISD::CopyToReg: {
1253 Register DestReg = cast<RegisterSDNode>(Val: Node->getOperand(Num: 1))->getReg();
1254 SDValue SrcVal = Node->getOperand(Num: 2);
1255 if (DestReg.isVirtual() && SrcVal.isMachineOpcode() &&
1256 SrcVal.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF) {
1257 // Instead building a COPY to that vreg destination, build an
1258 // IMPLICIT_DEF instruction instead.
1259 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Node->getDebugLoc(),
1260 MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg);
1261 break;
1262 }
1263 Register SrcReg;
1264 if (RegisterSDNode *R = dyn_cast<RegisterSDNode>(Val&: SrcVal))
1265 SrcReg = R->getReg();
1266 else
1267 SrcReg = getVR(Op: SrcVal, VRBaseMap);
1268
1269 if (SrcReg == DestReg) // Coalesced away the copy? Ignore.
1270 break;
1271
1272 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Node->getDebugLoc(), MCID: TII->get(Opcode: TargetOpcode::COPY),
1273 DestReg).addReg(RegNo: SrcReg);
1274 break;
1275 }
1276 case ISD::CopyFromReg: {
1277 unsigned SrcReg = cast<RegisterSDNode>(Val: Node->getOperand(Num: 1))->getReg();
1278 EmitCopyFromReg(Node, ResNo: 0, IsClone, SrcReg, VRBaseMap);
1279 break;
1280 }
1281 case ISD::EH_LABEL:
1282 case ISD::ANNOTATION_LABEL: {
1283 unsigned Opc = (Node->getOpcode() == ISD::EH_LABEL)
1284 ? TargetOpcode::EH_LABEL
1285 : TargetOpcode::ANNOTATION_LABEL;
1286 MCSymbol *S = cast<LabelSDNode>(Val: Node)->getLabel();
1287 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Node->getDebugLoc(),
1288 MCID: TII->get(Opcode: Opc)).addSym(Sym: S);
1289 break;
1290 }
1291
1292 case ISD::LIFETIME_START:
1293 case ISD::LIFETIME_END: {
1294 unsigned TarOp = (Node->getOpcode() == ISD::LIFETIME_START)
1295 ? TargetOpcode::LIFETIME_START
1296 : TargetOpcode::LIFETIME_END;
1297 auto *FI = cast<FrameIndexSDNode>(Val: Node->getOperand(Num: 1));
1298 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Node->getDebugLoc(), MCID: TII->get(Opcode: TarOp))
1299 .addFrameIndex(Idx: FI->getIndex());
1300 break;
1301 }
1302
1303 case ISD::PSEUDO_PROBE: {
1304 unsigned TarOp = TargetOpcode::PSEUDO_PROBE;
1305 auto Guid = cast<PseudoProbeSDNode>(Val: Node)->getGuid();
1306 auto Index = cast<PseudoProbeSDNode>(Val: Node)->getIndex();
1307 auto Attr = cast<PseudoProbeSDNode>(Val: Node)->getAttributes();
1308
1309 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Node->getDebugLoc(), MCID: TII->get(Opcode: TarOp))
1310 .addImm(Val: Guid)
1311 .addImm(Val: Index)
1312 .addImm(Val: (uint8_t)PseudoProbeType::Block)
1313 .addImm(Val: Attr);
1314 break;
1315 }
1316
1317 case ISD::INLINEASM:
1318 case ISD::INLINEASM_BR: {
1319 unsigned NumOps = Node->getNumOperands();
1320 if (Node->getOperand(Num: NumOps-1).getValueType() == MVT::Glue)
1321 --NumOps; // Ignore the glue operand.
1322
1323 // Create the inline asm machine instruction.
1324 unsigned TgtOpc = Node->getOpcode() == ISD::INLINEASM_BR
1325 ? TargetOpcode::INLINEASM_BR
1326 : TargetOpcode::INLINEASM;
1327 MachineInstrBuilder MIB =
1328 BuildMI(MF&: *MF, MIMD: Node->getDebugLoc(), MCID: TII->get(Opcode: TgtOpc));
1329
1330 // Add the asm string as an external symbol operand.
1331 SDValue AsmStrV = Node->getOperand(Num: InlineAsm::Op_AsmString);
1332 const char *AsmStr = cast<ExternalSymbolSDNode>(Val&: AsmStrV)->getSymbol();
1333 MIB.addExternalSymbol(FnName: AsmStr);
1334
1335 // Add the HasSideEffect, isAlignStack, AsmDialect, MayLoad and MayStore
1336 // bits.
1337 int64_t ExtraInfo =
1338 cast<ConstantSDNode>(Val: Node->getOperand(Num: InlineAsm::Op_ExtraInfo))->
1339 getZExtValue();
1340 MIB.addImm(Val: ExtraInfo);
1341
1342 // Remember to operand index of the group flags.
1343 SmallVector<unsigned, 8> GroupIdx;
1344
1345 // Remember registers that are part of early-clobber defs.
1346 SmallVector<unsigned, 8> ECRegs;
1347
1348 // Add all of the operand registers to the instruction.
1349 for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
1350 unsigned Flags = Node->getConstantOperandVal(Num: i);
1351 const InlineAsm::Flag F(Flags);
1352 const unsigned NumVals = F.getNumOperandRegisters();
1353
1354 GroupIdx.push_back(Elt: MIB->getNumOperands());
1355 MIB.addImm(Val: Flags);
1356 ++i; // Skip the ID value.
1357
1358 switch (F.getKind()) {
1359 case InlineAsm::Kind::RegDef:
1360 for (unsigned j = 0; j != NumVals; ++j, ++i) {
1361 Register Reg = cast<RegisterSDNode>(Val: Node->getOperand(Num: i))->getReg();
1362 // FIXME: Add dead flags for physical and virtual registers defined.
1363 // For now, mark physical register defs as implicit to help fast
1364 // regalloc. This makes inline asm look a lot like calls.
1365 MIB.addReg(RegNo: Reg, flags: RegState::Define | getImplRegState(B: Reg.isPhysical()));
1366 }
1367 break;
1368 case InlineAsm::Kind::RegDefEarlyClobber:
1369 case InlineAsm::Kind::Clobber:
1370 for (unsigned j = 0; j != NumVals; ++j, ++i) {
1371 Register Reg = cast<RegisterSDNode>(Val: Node->getOperand(Num: i))->getReg();
1372 MIB.addReg(RegNo: Reg, flags: RegState::Define | RegState::EarlyClobber |
1373 getImplRegState(B: Reg.isPhysical()));
1374 ECRegs.push_back(Elt: Reg);
1375 }
1376 break;
1377 case InlineAsm::Kind::RegUse: // Use of register.
1378 case InlineAsm::Kind::Imm: // Immediate.
1379 case InlineAsm::Kind::Mem: // Non-function addressing mode.
1380 // The addressing mode has been selected, just add all of the
1381 // operands to the machine instruction.
1382 for (unsigned j = 0; j != NumVals; ++j, ++i)
1383 AddOperand(MIB, Op: Node->getOperand(Num: i), IIOpNum: 0, II: nullptr, VRBaseMap,
1384 /*IsDebug=*/false, IsClone, IsCloned);
1385
1386 // Manually set isTied bits.
1387 if (F.isRegUseKind()) {
1388 unsigned DefGroup;
1389 if (F.isUseOperandTiedToDef(Idx&: DefGroup)) {
1390 unsigned DefIdx = GroupIdx[DefGroup] + 1;
1391 unsigned UseIdx = GroupIdx.back() + 1;
1392 for (unsigned j = 0; j != NumVals; ++j)
1393 MIB->tieOperands(DefIdx: DefIdx + j, UseIdx: UseIdx + j);
1394 }
1395 }
1396 break;
1397 case InlineAsm::Kind::Func: // Function addressing mode.
1398 for (unsigned j = 0; j != NumVals; ++j, ++i) {
1399 SDValue Op = Node->getOperand(Num: i);
1400 AddOperand(MIB, Op, IIOpNum: 0, II: nullptr, VRBaseMap,
1401 /*IsDebug=*/false, IsClone, IsCloned);
1402
1403 // Adjust Target Flags for function reference.
1404 if (auto *TGA = dyn_cast<GlobalAddressSDNode>(Val&: Op)) {
1405 unsigned NewFlags =
1406 MF->getSubtarget().classifyGlobalFunctionReference(
1407 GV: TGA->getGlobal());
1408 unsigned LastIdx = MIB.getInstr()->getNumOperands() - 1;
1409 MIB.getInstr()->getOperand(i: LastIdx).setTargetFlags(NewFlags);
1410 }
1411 }
1412 }
1413 }
1414
1415 // Add rounding control registers as implicit def for inline asm.
1416 if (MF->getFunction().hasFnAttribute(Kind: Attribute::StrictFP)) {
1417 ArrayRef<MCPhysReg> RCRegs = TLI->getRoundingControlRegisters();
1418 for (MCPhysReg Reg : RCRegs)
1419 MIB.addReg(RegNo: Reg, flags: RegState::ImplicitDefine);
1420 }
1421
1422 // GCC inline assembly allows input operands to also be early-clobber
1423 // output operands (so long as the operand is written only after it's
1424 // used), but this does not match the semantics of our early-clobber flag.
1425 // If an early-clobber operand register is also an input operand register,
1426 // then remove the early-clobber flag.
1427 for (unsigned Reg : ECRegs) {
1428 if (MIB->readsRegister(Reg, TRI)) {
1429 MachineOperand *MO =
1430 MIB->findRegisterDefOperand(Reg, TRI, isDead: false, Overlap: false);
1431 assert(MO && "No def operand for clobbered register?");
1432 MO->setIsEarlyClobber(false);
1433 }
1434 }
1435
1436 // Get the mdnode from the asm if it exists and add it to the instruction.
1437 SDValue MDV = Node->getOperand(Num: InlineAsm::Op_MDNode);
1438 const MDNode *MD = cast<MDNodeSDNode>(Val&: MDV)->getMD();
1439 if (MD)
1440 MIB.addMetadata(MD);
1441
1442 MBB->insert(I: InsertPos, MI: MIB);
1443 break;
1444 }
1445 }
1446}
1447
1448/// InstrEmitter - Construct an InstrEmitter and set it to start inserting
1449/// at the given position in the given block.
1450InstrEmitter::InstrEmitter(const TargetMachine &TM, MachineBasicBlock *mbb,
1451 MachineBasicBlock::iterator insertpos)
1452 : MF(mbb->getParent()), MRI(&MF->getRegInfo()),
1453 TII(MF->getSubtarget().getInstrInfo()),
1454 TRI(MF->getSubtarget().getRegisterInfo()),
1455 TLI(MF->getSubtarget().getTargetLowering()), MBB(mbb),
1456 InsertPos(insertpos) {
1457 EmitDebugInstrRefs = mbb->getParent()->useDebugInstrRef();
1458}
1459