1//==--- InstrEmitter.cpp - Emit MachineInstrs for the SelectionDAG class ---==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the Emit routines for the SelectionDAG class, which creates
10// MachineInstrs based on the decisions of the SelectionDAG instruction
11// selection.
12//
13//===----------------------------------------------------------------------===//
14
15#include "InstrEmitter.h"
16#include "SDNodeDbgValue.h"
17#include "llvm/BinaryFormat/Dwarf.h"
18#include "llvm/CodeGen/ISDOpcodes.h"
19#include "llvm/CodeGen/MachineConstantPool.h"
20#include "llvm/CodeGen/MachineFunction.h"
21#include "llvm/CodeGen/MachineInstrBuilder.h"
22#include "llvm/CodeGen/MachineRegisterInfo.h"
23#include "llvm/CodeGen/SelectionDAGNodes.h"
24#include "llvm/CodeGen/StackMaps.h"
25#include "llvm/CodeGen/TargetInstrInfo.h"
26#include "llvm/CodeGen/TargetLowering.h"
27#include "llvm/CodeGen/TargetSubtargetInfo.h"
28#include "llvm/IR/DebugInfoMetadata.h"
29#include "llvm/IR/PseudoProbe.h"
30#include "llvm/Support/ErrorHandling.h"
31#include "llvm/Target/TargetMachine.h"
32using namespace llvm;
33
34#define DEBUG_TYPE "instr-emitter"
35
36/// MinRCSize - Smallest register class we allow when constraining virtual
37/// registers. If satisfying all register class constraints would require
38/// using a smaller register class, emit a COPY to a new virtual register
39/// instead.
40const unsigned MinRCSize = 4;
41
42/// CountResults - The results of target nodes have register or immediate
43/// operands first, then an optional chain, and optional glue operands (which do
44/// not go into the resulting MachineInstr).
45unsigned InstrEmitter::CountResults(SDNode *Node) {
46 unsigned N = Node->getNumValues();
47 while (N && Node->getValueType(ResNo: N - 1) == MVT::Glue)
48 --N;
49 if (N && Node->getValueType(ResNo: N - 1) == MVT::Other)
50 --N; // Skip over chain result.
51 return N;
52}
53
54/// countOperands - The inputs to target nodes have any actual inputs first,
55/// followed by an optional chain operand, then an optional glue operand.
56/// Compute the number of actual operands that will go into the resulting
57/// MachineInstr.
58///
59/// Also count physreg RegisterSDNode and RegisterMaskSDNode operands preceding
60/// the chain and glue. These operands may be implicit on the machine instr.
61static unsigned countOperands(SDNode *Node, unsigned NumExpUses,
62 unsigned &NumImpUses) {
63 unsigned N = Node->getNumOperands();
64 while (N && Node->getOperand(Num: N - 1).getValueType() == MVT::Glue)
65 --N;
66 if (N && Node->getOperand(Num: N - 1).getOpcode() == ISD::DEACTIVATION_SYMBOL)
67 --N; // Ignore deactivation symbol if it exists.
68 if (N && Node->getOperand(Num: N - 1).getValueType() == MVT::Other)
69 --N; // Ignore chain if it exists.
70
71 // Count RegisterSDNode and RegisterMaskSDNode operands for NumImpUses.
72 NumImpUses = N - NumExpUses;
73 for (unsigned I = N; I > NumExpUses; --I) {
74 if (isa<RegisterMaskSDNode>(Val: Node->getOperand(Num: I - 1)))
75 continue;
76 if (RegisterSDNode *RN = dyn_cast<RegisterSDNode>(Val: Node->getOperand(Num: I - 1)))
77 if (RN->getReg().isPhysical())
78 continue;
79 NumImpUses = N - I;
80 break;
81 }
82
83 return N;
84}
85
86/// EmitCopyFromReg - Generate machine code for an CopyFromReg node or an
87/// implicit physical register output.
88void InstrEmitter::EmitCopyFromReg(SDValue Op, bool IsClone, Register SrcReg,
89 VRBaseMapType &VRBaseMap) {
90 Register VRBase;
91 if (SrcReg.isVirtual()) {
92 // Just use the input register directly!
93 if (IsClone)
94 VRBaseMap.erase(Val: Op);
95 bool isNew = VRBaseMap.insert(KV: std::make_pair(x&: Op, y&: SrcReg)).second;
96 (void)isNew; // Silence compiler warning.
97 assert(isNew && "Node emitted out of order - early");
98 return;
99 }
100
101 // If the node is only used by a CopyToReg and the dest reg is a vreg, use
102 // the CopyToReg'd destination register instead of creating a new vreg.
103 bool MatchReg = true;
104
105 MVT VT = Op.getSimpleValueType();
106
107 // FIXME: The Untyped check is a workaround for SystemZ i128 inline assembly
108 // using i128, when it should probably be using v2i64.
109 const TargetRegisterClass *UseRC =
110 VT == MVT::Untyped ? nullptr : TLI->getRegClassFor(VT, isDivergent: Op->isDivergent());
111
112 for (SDNode *User : Op->users()) {
113 bool Match = true;
114 if (User->getOpcode() == ISD::CopyToReg && User->getOperand(Num: 2) == Op) {
115 Register DestReg = cast<RegisterSDNode>(Val: User->getOperand(Num: 1))->getReg();
116 if (DestReg.isVirtual()) {
117 VRBase = DestReg;
118 Match = false;
119 } else if (DestReg != SrcReg)
120 Match = false;
121 } else {
122 for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) {
123 if (User->getOperand(Num: i) != Op)
124 continue;
125 if (VT == MVT::Other || VT == MVT::Glue)
126 continue;
127 Match = false;
128 if (User->isMachineOpcode()) {
129 const MCInstrDesc &II = TII->get(Opcode: User->getMachineOpcode());
130 const TargetRegisterClass *RC = nullptr;
131 if (i + II.getNumDefs() < II.getNumOperands()) {
132 RC = TRI->getAllocatableClass(
133 RC: TII->getRegClass(MCID: II, OpNum: i + II.getNumDefs()));
134 }
135 if (!UseRC)
136 UseRC = RC;
137 else if (RC) {
138 const TargetRegisterClass *ComRC =
139 TRI->getCommonSubClass(A: UseRC, B: RC);
140 // If multiple uses expect disjoint register classes, we emit
141 // copies in AddRegisterOperand.
142 if (ComRC)
143 UseRC = ComRC;
144 }
145 }
146 }
147 }
148 MatchReg &= Match;
149 if (VRBase)
150 break;
151 }
152
153 const TargetRegisterClass *SrcRC = nullptr, *DstRC = nullptr;
154 SrcRC = TRI->getMinimalPhysRegClass(Reg: SrcReg);
155
156 // Figure out the register class to create for the destreg.
157 if (VRBase) {
158 DstRC = MRI->getRegClass(Reg: VRBase);
159 } else if (UseRC) {
160 assert(TRI->isTypeLegalForClass(*UseRC, VT) &&
161 "Incompatible phys register def and uses!");
162 DstRC = UseRC;
163 } else
164 DstRC = SrcRC;
165
166 // If all uses are reading from the src physical register and copying the
167 // register is either impossible or very expensive, then don't create a copy.
168 if (MatchReg && SrcRC->expensiveOrImpossibleToCopy()) {
169 VRBase = SrcReg;
170 } else {
171 // Create the reg, emit the copy.
172 VRBase = MRI->createVirtualRegister(RegClass: DstRC);
173 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Op.getDebugLoc(), MCID: TII->get(Opcode: TargetOpcode::COPY),
174 DestReg: VRBase)
175 .addReg(RegNo: SrcReg);
176 }
177
178 if (IsClone)
179 VRBaseMap.erase(Val: Op);
180 bool isNew = VRBaseMap.insert(KV: std::make_pair(x&: Op, y&: VRBase)).second;
181 (void)isNew; // Silence compiler warning.
182 assert(isNew && "Node emitted out of order - early");
183}
184
185void InstrEmitter::CreateVirtualRegisters(SDNode *Node,
186 MachineInstrBuilder &MIB,
187 const MCInstrDesc &II,
188 bool IsClone, bool IsCloned,
189 VRBaseMapType &VRBaseMap) {
190 assert(Node->getMachineOpcode() != TargetOpcode::IMPLICIT_DEF &&
191 "IMPLICIT_DEF should have been handled as a special case elsewhere!");
192
193 unsigned NumResults = CountResults(Node);
194 bool HasVRegVariadicDefs = !MF->getTarget().usesPhysRegsForValues() &&
195 II.isVariadic() && II.variadicOpsAreDefs();
196 unsigned NumVRegs = HasVRegVariadicDefs ? NumResults : II.getNumDefs();
197 if (Node->getMachineOpcode() == TargetOpcode::STATEPOINT)
198 NumVRegs = NumResults;
199 for (unsigned i = 0; i < NumVRegs; ++i) {
200 // If the specific node value is only used by a CopyToReg and the dest reg
201 // is a vreg in the same register class, use the CopyToReg'd destination
202 // register instead of creating a new vreg.
203 Register VRBase;
204 const TargetRegisterClass *RC =
205 TRI->getAllocatableClass(RC: TII->getRegClass(MCID: II, OpNum: i));
206 // Always let the value type influence the used register class. The
207 // constraints on the instruction may be too lax to represent the value
208 // type correctly. For example, a 64-bit float (X86::FR64) can't live in
209 // the 32-bit float super-class (X86::FR32).
210 if (i < NumResults && TLI->isTypeLegal(VT: Node->getSimpleValueType(ResNo: i))) {
211 const TargetRegisterClass *VTRC = TLI->getRegClassFor(
212 VT: Node->getSimpleValueType(ResNo: i),
213 isDivergent: (Node->isDivergent() || (RC && TRI->isDivergentRegClass(RC))));
214 if (RC)
215 VTRC = TRI->getCommonSubClass(A: RC, B: VTRC);
216 if (VTRC)
217 RC = VTRC;
218 }
219
220 if (!II.operands().empty() && II.operands()[i].isOptionalDef()) {
221 // Optional def must be a physical register.
222 VRBase = cast<RegisterSDNode>(Val: Node->getOperand(Num: i-NumResults))->getReg();
223 assert(VRBase.isPhysical());
224 MIB.addReg(RegNo: VRBase, Flags: RegState::Define);
225 }
226
227 if (!VRBase && !IsClone && !IsCloned)
228 for (SDNode *User : Node->users()) {
229 if (User->getOpcode() == ISD::CopyToReg &&
230 User->getOperand(Num: 2).getNode() == Node &&
231 User->getOperand(Num: 2).getResNo() == i) {
232 Register Reg = cast<RegisterSDNode>(Val: User->getOperand(Num: 1))->getReg();
233 if (Reg.isVirtual()) {
234 const TargetRegisterClass *RegRC = MRI->getRegClass(Reg);
235 if (RegRC == RC) {
236 VRBase = Reg;
237 MIB.addReg(RegNo: VRBase, Flags: RegState::Define);
238 break;
239 }
240 }
241 }
242 }
243
244 // Create the result registers for this node and add the result regs to
245 // the machine instruction.
246 if (!VRBase) {
247 assert(RC && "Isn't a register operand!");
248 VRBase = MRI->createVirtualRegister(RegClass: RC);
249 MIB.addReg(RegNo: VRBase, Flags: RegState::Define);
250 }
251
252 // If this def corresponds to a result of the SDNode insert the VRBase into
253 // the lookup map.
254 if (i < NumResults) {
255 SDValue Op(Node, i);
256 if (IsClone)
257 VRBaseMap.erase(Val: Op);
258 bool isNew = VRBaseMap.insert(KV: std::make_pair(x&: Op, y&: VRBase)).second;
259 (void)isNew; // Silence compiler warning.
260 assert(isNew && "Node emitted out of order - early");
261 }
262 }
263}
264
265/// getVR - Return the virtual register corresponding to the specified result
266/// of the specified node.
267Register InstrEmitter::getVR(SDValue Op, VRBaseMapType &VRBaseMap) {
268 if (Op.isMachineOpcode() &&
269 Op.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF) {
270 // Add an IMPLICIT_DEF instruction before every use.
271 // IMPLICIT_DEF can produce any type of result so its MCInstrDesc
272 // does not include operand register class info.
273 const TargetRegisterClass *RC = TLI->getRegClassFor(
274 VT: Op.getSimpleValueType(), isDivergent: Op.getNode()->isDivergent());
275 Register VReg = MRI->createVirtualRegister(RegClass: RC);
276 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Op.getDebugLoc(),
277 MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: VReg);
278 return VReg;
279 }
280
281 VRBaseMapType::iterator I = VRBaseMap.find(Val: Op);
282 assert(I != VRBaseMap.end() && "Node emitted out of order - late");
283 return I->second;
284}
285
286/// AddRegisterOperand - Add the specified register as an operand to the
287/// specified machine instr. Insert register copies if the register is
288/// not in the required register class.
289void InstrEmitter::AddRegisterOperand(MachineInstrBuilder &MIB, SDValue Op,
290 unsigned IIOpNum, const MCInstrDesc *II,
291 VRBaseMapType &VRBaseMap, bool IsDebug) {
292 assert(Op.getValueType() != MVT::Other &&
293 Op.getValueType() != MVT::Glue &&
294 "Chain and glue operands should occur at end of operand list!");
295 // Get/emit the operand.
296 Register VReg = getVR(Op, VRBaseMap);
297
298 const MCInstrDesc &MCID = MIB->getDesc();
299 bool isOptDef = IIOpNum < MCID.getNumOperands() &&
300 MCID.operands()[IIOpNum].isOptionalDef();
301
302 // If the instruction requires a register in a different class, create
303 // a new virtual register and copy the value into it, but first attempt to
304 // shrink VReg's register class within reason. For example, if VReg == GR32
305 // and II requires a GR32_NOSP, just constrain VReg to GR32_NOSP.
306 if (II) {
307 const TargetRegisterClass *OpRC = nullptr;
308 if (IIOpNum < II->getNumOperands())
309 OpRC = TII->getRegClass(MCID: *II, OpNum: IIOpNum);
310
311 if (OpRC) {
312 unsigned MinNumRegs = MinRCSize;
313 // Don't apply any RC size limit for IMPLICIT_DEF. Each use has a unique
314 // virtual register.
315 if (Op.isMachineOpcode() &&
316 Op.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF)
317 MinNumRegs = 0;
318
319 const TargetRegisterClass *ConstrainedRC
320 = MRI->constrainRegClass(Reg: VReg, RC: OpRC, MinNumRegs);
321 if (!ConstrainedRC) {
322 OpRC = TRI->getAllocatableClass(RC: OpRC);
323 assert(OpRC && "Constraints cannot be fulfilled for allocation");
324 Register NewVReg = MRI->createVirtualRegister(RegClass: OpRC);
325 BuildMI(BB&: *MBB, I: InsertPos, MIMD: MIB->getDebugLoc(),
326 MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: NewVReg)
327 .addReg(RegNo: VReg);
328 VReg = NewVReg;
329 } else {
330 assert(ConstrainedRC->isAllocatable() &&
331 "Constraining an allocatable VReg produced an unallocatable class?");
332 }
333 }
334 }
335
336 MIB.addReg(RegNo: VReg, Flags: getDefRegState(B: isOptDef) | getDebugRegState(B: IsDebug));
337}
338
339/// AddOperand - Add the specified operand to the specified machine instr. II
340/// specifies the instruction information for the node, and IIOpNum is the
341/// operand number (in the II) that we are adding.
342void InstrEmitter::AddOperand(MachineInstrBuilder &MIB, SDValue Op,
343 unsigned IIOpNum, const MCInstrDesc *II,
344 VRBaseMapType &VRBaseMap, bool IsDebug,
345 bool IsClone, bool IsCloned) {
346 if (Op.isMachineOpcode()) {
347 AddRegisterOperand(MIB, Op, IIOpNum, II, VRBaseMap, IsDebug);
348 } else if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
349 if (C->getAPIntValue().getSignificantBits() <= 64) {
350 MIB.addImm(Val: C->getSExtValue());
351 } else {
352 MIB.addCImm(
353 Val: ConstantInt::get(Context&: MF->getFunction().getContext(), V: C->getAPIntValue()));
354 }
355 } else if (ConstantFPSDNode *F = dyn_cast<ConstantFPSDNode>(Val&: Op)) {
356 MIB.addFPImm(Val: F->getConstantFPValue());
357 } else if (RegisterSDNode *R = dyn_cast<RegisterSDNode>(Val&: Op)) {
358 Register VReg = R->getReg();
359 MVT OpVT = Op.getSimpleValueType();
360 const TargetRegisterClass *IIRC =
361 II ? TRI->getAllocatableClass(RC: TII->getRegClass(MCID: *II, OpNum: IIOpNum)) : nullptr;
362 const TargetRegisterClass *OpRC =
363 TLI->isTypeLegal(VT: OpVT)
364 ? TLI->getRegClassFor(VT: OpVT,
365 isDivergent: Op.getNode()->isDivergent() ||
366 (IIRC && TRI->isDivergentRegClass(RC: IIRC)))
367 : nullptr;
368
369 if (OpRC && IIRC && OpRC != IIRC && VReg.isVirtual()) {
370 Register NewVReg = MRI->createVirtualRegister(RegClass: IIRC);
371 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Op.getNode()->getDebugLoc(),
372 MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: NewVReg).addReg(RegNo: VReg);
373 VReg = NewVReg;
374 }
375 // Turn additional physreg operands into implicit uses on non-variadic
376 // instructions. This is used by call and return instructions passing
377 // arguments in registers.
378 bool Imp = II && (IIOpNum >= II->getNumOperands() && !II->isVariadic());
379 MIB.addReg(RegNo: VReg, Flags: getImplRegState(B: Imp));
380 } else if (RegisterMaskSDNode *RM = dyn_cast<RegisterMaskSDNode>(Val&: Op)) {
381 MIB.addRegMask(Mask: RM->getRegMask());
382 } else if (GlobalAddressSDNode *TGA = dyn_cast<GlobalAddressSDNode>(Val&: Op)) {
383 MIB.addGlobalAddress(GV: TGA->getGlobal(), Offset: TGA->getOffset(),
384 TargetFlags: TGA->getTargetFlags());
385 } else if (BasicBlockSDNode *BBNode = dyn_cast<BasicBlockSDNode>(Val&: Op)) {
386 MIB.addMBB(MBB: BBNode->getBasicBlock());
387 } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Op)) {
388 MIB.addFrameIndex(Idx: FI->getIndex());
389 } else if (JumpTableSDNode *JT = dyn_cast<JumpTableSDNode>(Val&: Op)) {
390 MIB.addJumpTableIndex(Idx: JT->getIndex(), TargetFlags: JT->getTargetFlags());
391 } else if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(Val&: Op)) {
392 int Offset = CP->getOffset();
393 Align Alignment = CP->getAlign();
394
395 unsigned Idx;
396 MachineConstantPool *MCP = MF->getConstantPool();
397 if (CP->isMachineConstantPoolEntry())
398 Idx = MCP->getConstantPoolIndex(V: CP->getMachineCPVal(), Alignment);
399 else
400 Idx = MCP->getConstantPoolIndex(C: CP->getConstVal(), Alignment);
401 MIB.addConstantPoolIndex(Idx, Offset, TargetFlags: CP->getTargetFlags());
402 } else if (ExternalSymbolSDNode *ES = dyn_cast<ExternalSymbolSDNode>(Val&: Op)) {
403 MIB.addExternalSymbol(FnName: ES->getSymbol(), TargetFlags: ES->getTargetFlags());
404 } else if (auto *SymNode = dyn_cast<MCSymbolSDNode>(Val&: Op)) {
405 MIB.addSym(Sym: SymNode->getMCSymbol());
406 } else if (BlockAddressSDNode *BA = dyn_cast<BlockAddressSDNode>(Val&: Op)) {
407 MIB.addBlockAddress(BA: BA->getBlockAddress(),
408 Offset: BA->getOffset(),
409 TargetFlags: BA->getTargetFlags());
410 } else if (TargetIndexSDNode *TI = dyn_cast<TargetIndexSDNode>(Val&: Op)) {
411 MIB.addTargetIndex(Idx: TI->getIndex(), Offset: TI->getOffset(), TargetFlags: TI->getTargetFlags());
412 } else {
413 assert(Op.getValueType() != MVT::Other &&
414 Op.getValueType() != MVT::Glue &&
415 "Chain and glue operands should occur at end of operand list!");
416 AddRegisterOperand(MIB, Op, IIOpNum, II, VRBaseMap, IsDebug);
417 }
418}
419
420Register InstrEmitter::ConstrainForSubReg(Register VReg, unsigned SubIdx,
421 MVT VT, bool isDivergent, const DebugLoc &DL) {
422 const TargetRegisterClass *VRC = MRI->getRegClass(Reg: VReg);
423 const TargetRegisterClass *RC = TRI->getSubClassWithSubReg(RC: VRC, Idx: SubIdx);
424
425 // RC is a sub-class of VRC that supports SubIdx. Try to constrain VReg
426 // within reason.
427 if (RC && RC != VRC)
428 RC = MRI->constrainRegClass(Reg: VReg, RC, MinNumRegs: MinRCSize);
429
430 // VReg has been adjusted. It can be used with SubIdx operands now.
431 if (RC)
432 return VReg;
433
434 // VReg couldn't be reasonably constrained. Emit a COPY to a new virtual
435 // register instead.
436 RC = TRI->getSubClassWithSubReg(RC: TLI->getRegClassFor(VT, isDivergent), Idx: SubIdx);
437 assert(RC && "No legal register class for VT supports that SubIdx");
438 Register NewReg = MRI->createVirtualRegister(RegClass: RC);
439 BuildMI(BB&: *MBB, I: InsertPos, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: NewReg)
440 .addReg(RegNo: VReg);
441 return NewReg;
442}
443
444/// EmitSubregNode - Generate machine code for subreg nodes.
445///
446void InstrEmitter::EmitSubregNode(SDNode *Node, VRBaseMapType &VRBaseMap,
447 bool IsClone, bool IsCloned) {
448 Register VRBase;
449 unsigned Opc = Node->getMachineOpcode();
450
451 // If the node is only used by a CopyToReg and the dest reg is a vreg, use
452 // the CopyToReg'd destination register instead of creating a new vreg.
453 for (SDNode *User : Node->users()) {
454 if (User->getOpcode() == ISD::CopyToReg &&
455 User->getOperand(Num: 2).getNode() == Node) {
456 Register DestReg = cast<RegisterSDNode>(Val: User->getOperand(Num: 1))->getReg();
457 if (DestReg.isVirtual()) {
458 VRBase = DestReg;
459 break;
460 }
461 }
462 }
463
464 if (Opc == TargetOpcode::EXTRACT_SUBREG) {
465 // EXTRACT_SUBREG is lowered as %dst = COPY %src:sub. There are no
466 // constraints on the %dst register, COPY can target all legal register
467 // classes.
468 unsigned SubIdx = Node->getConstantOperandVal(Num: 1);
469 const TargetRegisterClass *TRC =
470 TLI->getRegClassFor(VT: Node->getSimpleValueType(ResNo: 0), isDivergent: Node->isDivergent());
471
472 Register Reg;
473 MachineInstr *DefMI;
474 RegisterSDNode *R = dyn_cast<RegisterSDNode>(Val: Node->getOperand(Num: 0));
475 if (R && R->getReg().isPhysical()) {
476 Reg = R->getReg();
477 DefMI = nullptr;
478 } else {
479 Reg = R ? R->getReg() : getVR(Op: Node->getOperand(Num: 0), VRBaseMap);
480 DefMI = MRI->getVRegDef(Reg);
481 }
482
483 Register SrcReg, DstReg;
484 unsigned DefSubIdx;
485 if (DefMI &&
486 TII->isCoalescableExtInstr(MI: *DefMI, SrcReg, DstReg, SubIdx&: DefSubIdx) &&
487 SubIdx == DefSubIdx &&
488 TRC == MRI->getRegClass(Reg: SrcReg)) {
489 // Optimize these:
490 // r1025 = s/zext r1024, 4
491 // r1026 = extract_subreg r1025, 4
492 // to a copy
493 // r1026 = copy r1024
494 VRBase = MRI->createVirtualRegister(RegClass: TRC);
495 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Node->getDebugLoc(),
496 MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: VRBase)
497 .addReg(RegNo: SrcReg);
498 } else {
499 // Reg may not support a SubIdx sub-register, and we may need to
500 // constrain its register class or issue a COPY to a compatible register
501 // class.
502 if (Reg.isVirtual())
503 Reg = ConstrainForSubReg(VReg: Reg, SubIdx,
504 VT: Node->getOperand(Num: 0).getSimpleValueType(),
505 isDivergent: Node->isDivergent(), DL: Node->getDebugLoc());
506 // Create the destreg if it is missing.
507 if (!VRBase)
508 VRBase = MRI->createVirtualRegister(RegClass: TRC);
509
510 // Create the extract_subreg machine instruction.
511 MachineInstrBuilder CopyMI =
512 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Node->getDebugLoc(),
513 MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: VRBase);
514 if (Reg.isVirtual())
515 CopyMI.addReg(RegNo: Reg, Flags: {}, SubReg: SubIdx);
516 else
517 CopyMI.addReg(RegNo: TRI->getSubReg(Reg, Idx: SubIdx));
518 }
519 } else if (Opc == TargetOpcode::INSERT_SUBREG ||
520 Opc == TargetOpcode::SUBREG_TO_REG) {
521 SDValue Reg;
522 SDValue SubReg;
523 unsigned SubIdx;
524 if (Opc == TargetOpcode::INSERT_SUBREG) {
525 Reg = Node->getOperand(Num: 0);
526 SubReg = Node->getOperand(Num: 1);
527 SubIdx = Node->getOperand(Num: 2)->getAsZExtVal();
528 } else {
529 SubReg = Node->getOperand(Num: 0);
530 SubIdx = Node->getOperand(Num: 1)->getAsZExtVal();
531 }
532
533 // Figure out the register class to create for the destreg. It should be
534 // the largest legal register class supporting SubIdx sub-registers.
535 // RegisterCoalescer will constrain it further if it decides to eliminate
536 // the INSERT_SUBREG instruction.
537 //
538 // %dst = INSERT_SUBREG %src, %sub, SubIdx
539 //
540 // is lowered by TwoAddressInstructionPass to:
541 //
542 // %dst = COPY %src
543 // %dst:SubIdx = COPY %sub
544 //
545 // There is no constraint on the %src register class.
546 //
547 const TargetRegisterClass *SRC =
548 TLI->getRegClassFor(VT: Node->getSimpleValueType(ResNo: 0), isDivergent: Node->isDivergent());
549 SRC = TRI->getSubClassWithSubReg(RC: SRC, Idx: SubIdx);
550 assert(SRC && "No register class supports VT and SubIdx for INSERT_SUBREG");
551
552 if (VRBase == 0 || !SRC->hasSubClassEq(RC: MRI->getRegClass(Reg: VRBase)))
553 VRBase = MRI->createVirtualRegister(RegClass: SRC);
554
555 // Create the insert_subreg or subreg_to_reg machine instruction.
556 MachineInstrBuilder MIB =
557 BuildMI(MF&: *MF, MIMD: Node->getDebugLoc(), MCID: TII->get(Opcode: Opc), DestReg: VRBase);
558
559 // If creating an insert_subreg, then the first input operand
560 // is a register
561 if (Reg) {
562 AddOperand(MIB, Op: Reg, IIOpNum: 0, II: nullptr, VRBaseMap, /*IsDebug=*/false, IsClone,
563 IsCloned);
564 }
565 // Add the subregister being inserted
566 AddOperand(MIB, Op: SubReg, IIOpNum: 0, II: nullptr, VRBaseMap, /*IsDebug=*/false, IsClone,
567 IsCloned);
568 MIB.addImm(Val: SubIdx);
569 MBB->insert(I: InsertPos, MI: MIB);
570 } else
571 llvm_unreachable("Node is not insert_subreg, extract_subreg, or subreg_to_reg");
572
573 SDValue Op(Node, 0);
574 bool isNew = VRBaseMap.insert(KV: std::make_pair(x&: Op, y&: VRBase)).second;
575 (void)isNew; // Silence compiler warning.
576 assert(isNew && "Node emitted out of order - early");
577}
578
579/// EmitCopyToRegClassNode - Generate machine code for COPY_TO_REGCLASS nodes.
580/// COPY_TO_REGCLASS is just a normal copy, except that the destination
581/// register is constrained to be in a particular register class.
582///
583void
584InstrEmitter::EmitCopyToRegClassNode(SDNode *Node,
585 VRBaseMapType &VRBaseMap) {
586 // Create the new VReg in the destination class and emit a copy.
587 unsigned DstRCIdx = Node->getConstantOperandVal(Num: 1);
588 const TargetRegisterClass *DstRC =
589 TRI->getAllocatableClass(RC: TRI->getRegClass(i: DstRCIdx));
590 Register NewVReg = MRI->createVirtualRegister(RegClass: DstRC);
591 const MCInstrDesc &II = TII->get(Opcode: TargetOpcode::COPY);
592 MachineInstrBuilder MIB = BuildMI(MF&: *MF, MIMD: Node->getDebugLoc(), MCID: II, DestReg: NewVReg);
593 AddOperand(MIB, Op: Node->getOperand(Num: 0), IIOpNum: 1, II: &II, VRBaseMap, /*IsDebug=*/false,
594 /*IsClone=*/false, /*IsCloned*/ false);
595
596 MBB->insert(I: InsertPos, MI: MIB);
597 SDValue Op(Node, 0);
598 bool isNew = VRBaseMap.insert(KV: std::make_pair(x&: Op, y&: NewVReg)).second;
599 (void)isNew; // Silence compiler warning.
600 assert(isNew && "Node emitted out of order - early");
601}
602
603/// EmitRegSequence - Generate machine code for REG_SEQUENCE nodes.
604///
605void InstrEmitter::EmitRegSequence(SDNode *Node, VRBaseMapType &VRBaseMap,
606 bool IsClone, bool IsCloned) {
607 unsigned DstRCIdx = Node->getConstantOperandVal(Num: 0);
608 const TargetRegisterClass *RC = TRI->getRegClass(i: DstRCIdx);
609 Register NewVReg = MRI->createVirtualRegister(RegClass: TRI->getAllocatableClass(RC));
610 const MCInstrDesc &II = TII->get(Opcode: TargetOpcode::REG_SEQUENCE);
611 MachineInstrBuilder MIB = BuildMI(MF&: *MF, MIMD: Node->getDebugLoc(), MCID: II, DestReg: NewVReg);
612 unsigned NumOps = Node->getNumOperands();
613 // If the input pattern has a chain, then the root of the corresponding
614 // output pattern will get a chain as well. This can happen to be a
615 // REG_SEQUENCE (which is not "guarded" by countOperands/CountResults).
616 if (NumOps && Node->getOperand(Num: NumOps-1).getValueType() == MVT::Other)
617 --NumOps; // Ignore chain if it exists.
618
619 assert((NumOps & 1) == 1 &&
620 "REG_SEQUENCE must have an odd number of operands!");
621 for (unsigned i = 1; i != NumOps; ++i) {
622 SDValue Op = Node->getOperand(Num: i);
623 if ((i & 1) == 0) {
624 RegisterSDNode *R = dyn_cast<RegisterSDNode>(Val: Node->getOperand(Num: i-1));
625 // Skip physical registers as they don't have a vreg to get and we'll
626 // insert copies for them in TwoAddressInstructionPass anyway.
627 if (!R || !R->getReg().isPhysical()) {
628 unsigned SubIdx = Op->getAsZExtVal();
629 Register SubReg = getVR(Op: Node->getOperand(Num: i - 1), VRBaseMap);
630 const TargetRegisterClass *TRC = MRI->getRegClass(Reg: SubReg);
631 const TargetRegisterClass *SRC =
632 TRI->getMatchingSuperRegClass(A: RC, B: TRC, Idx: SubIdx);
633 if (SRC && SRC != RC) {
634 MRI->setRegClass(Reg: NewVReg, RC: SRC);
635 RC = SRC;
636 }
637 }
638 }
639 AddOperand(MIB, Op, IIOpNum: i+1, II: &II, VRBaseMap, /*IsDebug=*/false,
640 IsClone, IsCloned);
641 }
642
643 MBB->insert(I: InsertPos, MI: MIB);
644 SDValue Op(Node, 0);
645 bool isNew = VRBaseMap.insert(KV: std::make_pair(x&: Op, y&: NewVReg)).second;
646 (void)isNew; // Silence compiler warning.
647 assert(isNew && "Node emitted out of order - early");
648}
649
650/// EmitDbgValue - Generate machine instruction for a dbg_value node.
651///
652MachineInstr *InstrEmitter::EmitDbgValue(SDDbgValue *SD,
653 VRBaseMapType &VRBaseMap) {
654 assert(cast<DILocalVariable>(SD->getVariable())
655 ->isValidLocationForIntrinsic(SD->getDebugLoc()) &&
656 "Expected inlined-at fields to agree");
657
658 SD->setIsEmitted();
659
660 assert(!SD->getLocationOps().empty() &&
661 "dbg_value with no location operands?");
662
663 if (SD->isInvalidated())
664 return EmitDbgNoLocation(SD);
665
666 // Attempt to produce a DBG_INSTR_REF if we've been asked to.
667 if (EmitDebugInstrRefs)
668 if (auto *InstrRef = EmitDbgInstrRef(SD, VRBaseMap))
669 return InstrRef;
670
671 // Emit variadic dbg_value nodes as DBG_VALUE_LIST if they have not been
672 // emitted as instruction references.
673 if (SD->isVariadic())
674 return EmitDbgValueList(SD, VRBaseMap);
675
676 // Emit single-location dbg_value nodes as DBG_VALUE if they have not been
677 // emitted as instruction references.
678 return EmitDbgValueFromSingleOp(SD, VRBaseMap);
679}
680
681MachineOperand GetMOForConstDbgOp(const SDDbgOperand &Op) {
682 const Value *V = Op.getConst();
683 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: V)) {
684 if (CI->getBitWidth() > 64)
685 return MachineOperand::CreateCImm(CI);
686 if (CI->getBitWidth() == 1)
687 return MachineOperand::CreateImm(Val: CI->getZExtValue());
688 return MachineOperand::CreateImm(Val: CI->getSExtValue());
689 }
690 if (const ConstantFP *CF = dyn_cast<ConstantFP>(Val: V))
691 return MachineOperand::CreateFPImm(CFP: CF);
692 // Note: This assumes that all nullptr constants are zero-valued.
693 if (isa<ConstantPointerNull>(Val: V))
694 return MachineOperand::CreateImm(Val: 0);
695 // Undef or unhandled value type, so return an undef operand.
696 return MachineOperand::CreateReg(
697 /* Reg */ 0U, /* isDef */ false, /* isImp */ false,
698 /* isKill */ false, /* isDead */ false,
699 /* isUndef */ false, /* isEarlyClobber */ false,
700 /* SubReg */ 0, /* isDebug */ true);
701}
702
703void InstrEmitter::AddDbgValueLocationOps(
704 MachineInstrBuilder &MIB, const MCInstrDesc &DbgValDesc,
705 ArrayRef<SDDbgOperand> LocationOps,
706 VRBaseMapType &VRBaseMap) {
707 for (const SDDbgOperand &Op : LocationOps) {
708 switch (Op.getKind()) {
709 case SDDbgOperand::FRAMEIX:
710 MIB.addFrameIndex(Idx: Op.getFrameIx());
711 break;
712 case SDDbgOperand::VREG:
713 MIB.addReg(RegNo: Op.getVReg());
714 break;
715 case SDDbgOperand::SDNODE: {
716 SDValue V = SDValue(Op.getSDNode(), Op.getResNo());
717 // It's possible we replaced this SDNode with other(s) and therefore
718 // didn't generate code for it. It's better to catch these cases where
719 // they happen and transfer the debug info, but trying to guarantee that
720 // in all cases would be very fragile; this is a safeguard for any
721 // that were missed.
722 if (VRBaseMap.count(Val: V) == 0)
723 MIB.addReg(RegNo: 0U); // undef
724 else
725 AddOperand(MIB, Op: V, IIOpNum: (*MIB).getNumOperands(), II: &DbgValDesc, VRBaseMap,
726 /*IsDebug=*/true, /*IsClone=*/false, /*IsCloned=*/false);
727 } break;
728 case SDDbgOperand::CONST:
729 MIB.add(MO: GetMOForConstDbgOp(Op));
730 break;
731 case SDDbgOperand::GLOBALADDR:
732 MIB.addGlobalAddress(GV: Op.getGlobal());
733 break;
734 }
735 }
736}
737
738MachineInstr *
739InstrEmitter::EmitDbgInstrRef(SDDbgValue *SD,
740 VRBaseMapType &VRBaseMap) {
741 MDNode *Var = SD->getVariable();
742 const DIExpression *Expr = SD->getExpression();
743 DebugLoc DL = SD->getDebugLoc();
744 const MCInstrDesc &RefII = TII->get(Opcode: TargetOpcode::DBG_INSTR_REF);
745
746 // Returns true if the given operand is not a legal debug operand for a
747 // DBG_INSTR_REF.
748 auto IsInvalidOp = [](SDDbgOperand DbgOp) {
749 return DbgOp.getKind() == SDDbgOperand::FRAMEIX;
750 };
751 // Returns true if the given operand is not itself an instruction reference
752 // but is a legal debug operand for a DBG_INSTR_REF.
753 auto IsNonInstrRefOp = [](SDDbgOperand DbgOp) {
754 return DbgOp.getKind() == SDDbgOperand::CONST ||
755 DbgOp.getKind() == SDDbgOperand::GLOBALADDR;
756 };
757
758 // If this variable location does not depend on any instructions or contains
759 // any stack locations, produce it as a standard debug value instead.
760 if (any_of(Range: SD->getLocationOps(), P: IsInvalidOp) ||
761 all_of(Range: SD->getLocationOps(), P: IsNonInstrRefOp)) {
762 if (SD->isVariadic())
763 return EmitDbgValueList(SD, VRBaseMap);
764 return EmitDbgValueFromSingleOp(SD, VRBaseMap);
765 }
766
767 // Immediately fold any indirectness from the LLVM-IR intrinsic into the
768 // expression:
769 if (SD->isIndirect())
770 Expr = DIExpression::append(Expr, Ops: dwarf::DW_OP_deref);
771 // If this is not already a variadic expression, it must be modified to become
772 // one.
773 if (!SD->isVariadic())
774 Expr = DIExpression::convertToVariadicExpression(Expr);
775
776 SmallVector<MachineOperand> MOs;
777
778 // It may not be immediately possible to identify the MachineInstr that
779 // defines a VReg, it can depend for example on the order blocks are
780 // emitted in. When this happens, or when further analysis is needed later,
781 // produce an instruction like this:
782 //
783 // DBG_INSTR_REF !123, !456, %0:gr64
784 //
785 // i.e., point the instruction at the vreg, and patch it up later in
786 // MachineFunction::finalizeDebugInstrRefs.
787 auto AddVRegOp = [&](Register VReg) {
788 MOs.push_back(Elt: MachineOperand::CreateReg(
789 /* Reg */ VReg, /* isDef */ false, /* isImp */ false,
790 /* isKill */ false, /* isDead */ false,
791 /* isUndef */ false, /* isEarlyClobber */ false,
792 /* SubReg */ 0, /* isDebug */ true));
793 };
794 unsigned OpCount = SD->getLocationOps().size();
795 for (unsigned OpIdx = 0; OpIdx < OpCount; ++OpIdx) {
796 SDDbgOperand DbgOperand = SD->getLocationOps()[OpIdx];
797
798 // Try to find both the defined register and the instruction defining it.
799 MachineInstr *DefMI = nullptr;
800 Register VReg;
801
802 if (DbgOperand.getKind() == SDDbgOperand::VREG) {
803 VReg = DbgOperand.getVReg();
804
805 // No definition means that block hasn't been emitted yet. Leave a vreg
806 // reference to be fixed later.
807 if (!MRI->hasOneDef(RegNo: VReg)) {
808 AddVRegOp(VReg);
809 continue;
810 }
811
812 DefMI = &*MRI->def_instr_begin(RegNo: VReg);
813 } else if (DbgOperand.getKind() == SDDbgOperand::SDNODE) {
814 // Look up the corresponding VReg for the given SDNode, if any.
815 SDNode *Node = DbgOperand.getSDNode();
816 SDValue Op = SDValue(Node, DbgOperand.getResNo());
817 VRBaseMapType::iterator I = VRBaseMap.find(Val: Op);
818 // No VReg -> produce a DBG_VALUE $noreg instead.
819 if (I == VRBaseMap.end())
820 break;
821
822 // Try to pick out a defining instruction at this point.
823 VReg = getVR(Op, VRBaseMap);
824
825 // Again, if there's no instruction defining the VReg right now, fix it up
826 // later.
827 if (!MRI->hasOneDef(RegNo: VReg)) {
828 AddVRegOp(VReg);
829 continue;
830 }
831
832 DefMI = &*MRI->def_instr_begin(RegNo: VReg);
833 } else if (DbgOperand.getKind() == SDDbgOperand::GLOBALADDR) {
834 MOs.push_back(Elt: MachineOperand::CreateGA(GV: DbgOperand.getGlobal(),
835 /*Offset=*/0));
836 continue;
837 } else {
838 assert(DbgOperand.getKind() == SDDbgOperand::CONST);
839 MOs.push_back(Elt: GetMOForConstDbgOp(Op: DbgOperand));
840 continue;
841 }
842
843 // Avoid copy like instructions: they don't define values, only move them.
844 // Leave a virtual-register reference until it can be fixed up later, to
845 // find the underlying value definition.
846 if (DefMI->isCopyLike() || TII->isCopyInstr(MI: *DefMI)) {
847 AddVRegOp(VReg);
848 continue;
849 }
850
851 // Find the operand number which defines the specified VReg.
852 unsigned OperandIdx = 0;
853 for (const auto &MO : DefMI->operands()) {
854 if (MO.isReg() && MO.isDef() && MO.getReg() == VReg)
855 break;
856 ++OperandIdx;
857 }
858 assert(OperandIdx < DefMI->getNumOperands());
859
860 // Make the DBG_INSTR_REF refer to that instruction, and that operand.
861 unsigned InstrNum = DefMI->getDebugInstrNum();
862 MOs.push_back(Elt: MachineOperand::CreateDbgInstrRef(InstrIdx: InstrNum, OpIdx: OperandIdx));
863 }
864
865 // If we haven't created a valid MachineOperand for every DbgOp, abort and
866 // produce an undef DBG_VALUE.
867 if (MOs.size() != OpCount)
868 return EmitDbgNoLocation(SD);
869
870 return BuildMI(MF&: *MF, DL, MCID: RefII, IsIndirect: false, MOs, Variable: Var, Expr);
871}
872
873MachineInstr *InstrEmitter::EmitDbgNoLocation(SDDbgValue *SD) {
874 // An invalidated SDNode must generate an undef DBG_VALUE: although the
875 // original value is no longer computed, earlier DBG_VALUEs live ranges
876 // must not leak into later code.
877 DIVariable *Var = SD->getVariable();
878 const DIExpression *Expr =
879 DIExpression::convertToUndefExpression(Expr: SD->getExpression());
880 DebugLoc DL = SD->getDebugLoc();
881 const MCInstrDesc &Desc = TII->get(Opcode: TargetOpcode::DBG_VALUE);
882 return BuildMI(MF&: *MF, DL, MCID: Desc, IsIndirect: false, Reg: 0U, Variable: Var, Expr);
883}
884
885MachineInstr *
886InstrEmitter::EmitDbgValueList(SDDbgValue *SD,
887 VRBaseMapType &VRBaseMap) {
888 MDNode *Var = SD->getVariable();
889 DIExpression *Expr = SD->getExpression();
890 DebugLoc DL = SD->getDebugLoc();
891 // DBG_VALUE_LIST := "DBG_VALUE_LIST" var, expression, loc (, loc)*
892 const MCInstrDesc &DbgValDesc = TII->get(Opcode: TargetOpcode::DBG_VALUE_LIST);
893 // Build the DBG_VALUE_LIST instruction base.
894 auto MIB = BuildMI(MF&: *MF, MIMD: DL, MCID: DbgValDesc);
895 MIB.addMetadata(MD: Var);
896 MIB.addMetadata(MD: Expr);
897 AddDbgValueLocationOps(MIB, DbgValDesc, LocationOps: SD->getLocationOps(), VRBaseMap);
898 return &*MIB;
899}
900
901MachineInstr *
902InstrEmitter::EmitDbgValueFromSingleOp(SDDbgValue *SD,
903 VRBaseMapType &VRBaseMap) {
904 MDNode *Var = SD->getVariable();
905 DIExpression *Expr = SD->getExpression();
906 DebugLoc DL = SD->getDebugLoc();
907 const MCInstrDesc &II = TII->get(Opcode: TargetOpcode::DBG_VALUE);
908
909 assert(SD->getLocationOps().size() == 1 &&
910 "Non variadic dbg_value should have only one location op");
911
912 // See about constant-folding the expression.
913 // Copy the location operand in case we replace it.
914 SmallVector<SDDbgOperand, 1> LocationOps(1, SD->getLocationOps()[0]);
915 if (Expr && LocationOps[0].getKind() == SDDbgOperand::CONST) {
916 const Value *V = LocationOps[0].getConst();
917 if (auto *C = dyn_cast<ConstantInt>(Val: V)) {
918 std::tie(args&: Expr, args&: C) = Expr->constantFold(CI: C);
919 LocationOps[0] = SDDbgOperand::fromConst(Const: C);
920 }
921 }
922
923 // Emit non-variadic dbg_value nodes as DBG_VALUE.
924 // DBG_VALUE := "DBG_VALUE" loc, isIndirect, var, expr
925 auto MIB = BuildMI(MF&: *MF, MIMD: DL, MCID: II);
926 AddDbgValueLocationOps(MIB, DbgValDesc: II, LocationOps, VRBaseMap);
927
928 if (SD->isIndirect())
929 MIB.addImm(Val: 0U);
930 else
931 MIB.addReg(RegNo: 0U);
932
933 return MIB.addMetadata(MD: Var).addMetadata(MD: Expr);
934}
935
936MachineInstr *
937InstrEmitter::EmitDbgLabel(SDDbgLabel *SD) {
938 MDNode *Label = SD->getLabel();
939 DebugLoc DL = SD->getDebugLoc();
940 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) &&
941 "Expected inlined-at fields to agree");
942
943 const MCInstrDesc &II = TII->get(Opcode: TargetOpcode::DBG_LABEL);
944 MachineInstrBuilder MIB = BuildMI(MF&: *MF, MIMD: DL, MCID: II);
945 MIB.addMetadata(MD: Label);
946
947 return &*MIB;
948}
949
950/// EmitMachineNode - Generate machine code for a target-specific node and
951/// needed dependencies.
952///
953void InstrEmitter::
954EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
955 VRBaseMapType &VRBaseMap) {
956 unsigned Opc = Node->getMachineOpcode();
957
958 // Handle subreg insert/extract specially
959 if (Opc == TargetOpcode::EXTRACT_SUBREG ||
960 Opc == TargetOpcode::INSERT_SUBREG ||
961 Opc == TargetOpcode::SUBREG_TO_REG) {
962 EmitSubregNode(Node, VRBaseMap, IsClone, IsCloned);
963 return;
964 }
965
966 // Handle COPY_TO_REGCLASS specially.
967 if (Opc == TargetOpcode::COPY_TO_REGCLASS) {
968 EmitCopyToRegClassNode(Node, VRBaseMap);
969 return;
970 }
971
972 // Handle REG_SEQUENCE specially.
973 if (Opc == TargetOpcode::REG_SEQUENCE) {
974 EmitRegSequence(Node, VRBaseMap, IsClone, IsCloned);
975 return;
976 }
977
978 if (Opc == TargetOpcode::IMPLICIT_DEF)
979 // We want a unique VR for each IMPLICIT_DEF use.
980 return;
981
982 const MCInstrDesc &II = TII->get(Opcode: Opc);
983 unsigned NumResults = CountResults(Node);
984 unsigned NumDefs = II.getNumDefs();
985 const MCPhysReg *ScratchRegs = nullptr;
986
987 // Handle STACKMAP and PATCHPOINT specially and then use the generic code.
988 if (Opc == TargetOpcode::STACKMAP || Opc == TargetOpcode::PATCHPOINT) {
989 // Stackmaps do not have arguments and do not preserve their calling
990 // convention. However, to simplify runtime support, they clobber the same
991 // scratch registers as AnyRegCC.
992 unsigned CC = CallingConv::AnyReg;
993 if (Opc == TargetOpcode::PATCHPOINT) {
994 CC = Node->getConstantOperandVal(Num: PatchPointOpers::CCPos);
995 NumDefs = NumResults;
996 }
997 ScratchRegs = TLI->getScratchRegisters(CC: (CallingConv::ID) CC);
998 } else if (Opc == TargetOpcode::STATEPOINT) {
999 NumDefs = NumResults;
1000 }
1001
1002 unsigned NumImpUses = 0;
1003 unsigned NodeOperands =
1004 countOperands(Node, NumExpUses: II.getNumOperands() - NumDefs, NumImpUses);
1005 bool HasVRegVariadicDefs = !MF->getTarget().usesPhysRegsForValues() &&
1006 II.isVariadic() && II.variadicOpsAreDefs();
1007 bool HasPhysRegOuts = NumResults > NumDefs && !II.implicit_defs().empty() &&
1008 !HasVRegVariadicDefs;
1009#ifndef NDEBUG
1010 unsigned NumMIOperands = NodeOperands + NumResults;
1011 if (II.isVariadic())
1012 assert(NumMIOperands >= II.getNumOperands() &&
1013 "Too few operands for a variadic node!");
1014 else
1015 assert(NumMIOperands >= II.getNumOperands() &&
1016 NumMIOperands <=
1017 II.getNumOperands() + II.implicit_defs().size() + NumImpUses &&
1018 "#operands for dag node doesn't match .td file!");
1019#endif
1020
1021 // Create the new machine instruction.
1022 MachineInstrBuilder MIB = BuildMI(MF&: *MF, MIMD: Node->getDebugLoc(), MCID: II);
1023
1024 // Transfer IR flags from the SDNode to the MachineInstr
1025 MachineInstr *MI = MIB.getInstr();
1026 const SDNodeFlags Flags = Node->getFlags();
1027 if (Flags.hasUnpredictable())
1028 MI->setFlag(MachineInstr::MIFlag::Unpredictable);
1029
1030 // Add result register values for things that are defined by this
1031 // instruction.
1032 if (NumResults) {
1033 CreateVirtualRegisters(Node, MIB, II, IsClone, IsCloned, VRBaseMap);
1034
1035 if (Flags.hasNoSignedZeros())
1036 MI->setFlag(MachineInstr::MIFlag::FmNsz);
1037
1038 if (Flags.hasAllowReciprocal())
1039 MI->setFlag(MachineInstr::MIFlag::FmArcp);
1040
1041 if (Flags.hasNoNaNs())
1042 MI->setFlag(MachineInstr::MIFlag::FmNoNans);
1043
1044 if (Flags.hasNoInfs())
1045 MI->setFlag(MachineInstr::MIFlag::FmNoInfs);
1046
1047 if (Flags.hasAllowContract())
1048 MI->setFlag(MachineInstr::MIFlag::FmContract);
1049
1050 if (Flags.hasApproximateFuncs())
1051 MI->setFlag(MachineInstr::MIFlag::FmAfn);
1052
1053 if (Flags.hasAllowReassociation())
1054 MI->setFlag(MachineInstr::MIFlag::FmReassoc);
1055
1056 if (Flags.hasNoUnsignedWrap())
1057 MI->setFlag(MachineInstr::MIFlag::NoUWrap);
1058
1059 if (Flags.hasNoSignedWrap())
1060 MI->setFlag(MachineInstr::MIFlag::NoSWrap);
1061
1062 if (Flags.hasExact())
1063 MI->setFlag(MachineInstr::MIFlag::IsExact);
1064
1065 if (Flags.hasNoFPExcept())
1066 MI->setFlag(MachineInstr::MIFlag::NoFPExcept);
1067
1068 if (Flags.hasDisjoint())
1069 MI->setFlag(MachineInstr::MIFlag::Disjoint);
1070
1071 if (Flags.hasSameSign())
1072 MI->setFlag(MachineInstr::MIFlag::SameSign);
1073
1074 if (Flags.hasNoConvergent())
1075 MI->setFlag(MachineInstr::MIFlag::NoConvergent);
1076 }
1077
1078 // Emit all of the actual operands of this instruction, adding them to the
1079 // instruction as appropriate.
1080 bool HasOptPRefs = NumDefs > NumResults;
1081 assert((!HasOptPRefs || !HasPhysRegOuts) &&
1082 "Unable to cope with optional defs and phys regs defs!");
1083 unsigned NumSkip = HasOptPRefs ? NumDefs - NumResults : 0;
1084 for (unsigned i = NumSkip; i != NodeOperands; ++i)
1085 AddOperand(MIB, Op: Node->getOperand(Num: i), IIOpNum: i-NumSkip+NumDefs, II: &II,
1086 VRBaseMap, /*IsDebug=*/false, IsClone, IsCloned);
1087
1088 // Add scratch registers as implicit def and early clobber
1089 if (ScratchRegs)
1090 for (unsigned i = 0; ScratchRegs[i]; ++i)
1091 MIB.addReg(RegNo: ScratchRegs[i], Flags: RegState::ImplicitDefine |
1092 RegState::EarlyClobber);
1093
1094 // Set the memory reference descriptions of this instruction now that it is
1095 // part of the function.
1096 MIB.setMemRefs(cast<MachineSDNode>(Val: Node)->memoperands());
1097
1098 // Set the CFI type.
1099 MIB->setCFIType(MF&: *MF, Type: Node->getCFIType());
1100
1101 // Insert the instruction into position in the block. This needs to
1102 // happen before any custom inserter hook is called so that the
1103 // hook knows where in the block to insert the replacement code.
1104 MBB->insert(I: InsertPos, MI: MIB);
1105
1106 // The MachineInstr may also define physregs instead of virtregs. These
1107 // physreg values can reach other instructions in different ways:
1108 //
1109 // 1. When there is a use of a Node value beyond the explicitly defined
1110 // virtual registers, we emit a CopyFromReg for one of the implicitly
1111 // defined physregs. This only happens when HasPhysRegOuts is true.
1112 //
1113 // 2. A CopyFromReg reading a physreg may be glued to this instruction.
1114 //
1115 // 3. A glued instruction may implicitly use a physreg.
1116 //
1117 // 4. A glued instruction may use a RegisterSDNode operand.
1118 //
1119 // Collect all the used physreg defs, and make sure that any unused physreg
1120 // defs are marked as dead.
1121 SmallVector<Register, 8> UsedRegs;
1122
1123 // Additional results must be physical register defs.
1124 if (HasPhysRegOuts) {
1125 for (unsigned i = NumDefs; i < NumResults; ++i) {
1126 Register Reg = II.implicit_defs()[i - NumDefs];
1127 if (!Node->hasAnyUseOfValue(Value: i))
1128 continue;
1129 // This implicitly defined physreg has a use.
1130 UsedRegs.push_back(Elt: Reg);
1131 EmitCopyFromReg(Op: SDValue(Node, i), IsClone, SrcReg: Reg, VRBaseMap);
1132 }
1133 }
1134
1135 // Scan the glue chain for any used physregs.
1136 if (Node->getValueType(ResNo: Node->getNumValues()-1) == MVT::Glue) {
1137 for (SDNode *F = Node->getGluedUser(); F; F = F->getGluedUser()) {
1138 if (F->getOpcode() == ISD::CopyFromReg) {
1139 Register Reg = cast<RegisterSDNode>(Val: F->getOperand(Num: 1))->getReg();
1140 if (Reg.isPhysical())
1141 UsedRegs.push_back(Elt: Reg);
1142 continue;
1143 } else if (F->getOpcode() == ISD::CopyToReg) {
1144 // Skip CopyToReg nodes that are internal to the glue chain.
1145 continue;
1146 }
1147 // Collect declared implicit uses.
1148 const MCInstrDesc &MCID = TII->get(Opcode: F->getMachineOpcode());
1149 append_range(C&: UsedRegs, R: MCID.implicit_uses());
1150 // In addition to declared implicit uses, we must also check for
1151 // direct RegisterSDNode operands.
1152 for (const SDValue &Op : F->op_values())
1153 if (RegisterSDNode *R = dyn_cast<RegisterSDNode>(Val: Op)) {
1154 Register Reg = R->getReg();
1155 if (Reg.isPhysical())
1156 UsedRegs.push_back(Elt: Reg);
1157 }
1158 }
1159 }
1160
1161 // Add rounding control registers as implicit def for function call.
1162 if (II.isCall() && MF->getFunction().hasFnAttribute(Kind: Attribute::StrictFP)) {
1163 ArrayRef<MCPhysReg> RCRegs = TLI->getRoundingControlRegisters();
1164 llvm::append_range(C&: UsedRegs, R&: RCRegs);
1165 }
1166
1167 // Finally mark unused registers as dead.
1168 if (!UsedRegs.empty() || !II.implicit_defs().empty() || II.hasOptionalDef())
1169 MIB->setPhysRegsDeadExcept(UsedRegs, TRI: *TRI);
1170
1171 // STATEPOINT is too 'dynamic' to have meaningful machine description.
1172 // We have to manually tie operands.
1173 if (Opc == TargetOpcode::STATEPOINT && NumDefs > 0) {
1174 assert(!HasPhysRegOuts && "STATEPOINT mishandled");
1175 MachineInstr *MI = MIB;
1176 unsigned Def = 0;
1177 int First = StatepointOpers(MI).getFirstGCPtrIdx();
1178 assert(First > 0 && "Statepoint has Defs but no GC ptr list");
1179 unsigned Use = (unsigned)First;
1180 while (Def < NumDefs) {
1181 if (MI->getOperand(i: Use).isReg())
1182 MI->tieOperands(DefIdx: Def++, UseIdx: Use);
1183 Use = StackMaps::getNextMetaArgIdx(MI, CurIdx: Use);
1184 }
1185 }
1186
1187 unsigned Op = Node->getNumOperands();
1188 if (Op != 0 && Node->getOperand(Num: Op - 1)->getOpcode() ==
1189 ~(unsigned)TargetOpcode::CONVERGENCECTRL_GLUE) {
1190 Register VReg = getVR(Op: Node->getOperand(Num: Op - 1)->getOperand(Num: 0), VRBaseMap);
1191 MachineOperand MO = MachineOperand::CreateReg(Reg: VReg, /*isDef=*/false,
1192 /*isImp=*/true);
1193 MIB->addOperand(Op: MO);
1194 Op--;
1195 }
1196
1197 if (Op != 0 &&
1198 Node->getOperand(Num: Op - 1)->getOpcode() == ISD::DEACTIVATION_SYMBOL) {
1199 MI->setDeactivationSymbol(
1200 MF&: *MF, DS: const_cast<GlobalValue *>(
1201 cast<DeactivationSymbolSDNode>(Val: Node->getOperand(Num: Op - 1))
1202 ->getGlobal()));
1203 Op--;
1204 }
1205
1206 // Run post-isel target hook to adjust this instruction if needed.
1207 if (II.hasPostISelHook())
1208 TLI->AdjustInstrPostInstrSelection(MI&: *MIB, Node);
1209}
1210
1211/// EmitSpecialNode - Generate machine code for a target-independent node and
1212/// needed dependencies.
1213void InstrEmitter::
1214EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned,
1215 VRBaseMapType &VRBaseMap) {
1216 switch (Node->getOpcode()) {
1217 default:
1218#ifndef NDEBUG
1219 Node->dump();
1220#endif
1221 llvm_unreachable("This target-independent node should have been selected!");
1222 case ISD::EntryToken:
1223 case ISD::MERGE_VALUES:
1224 case ISD::TokenFactor:
1225 case ISD::DEACTIVATION_SYMBOL:
1226 break;
1227 case ISD::CopyToReg: {
1228 Register DestReg = cast<RegisterSDNode>(Val: Node->getOperand(Num: 1))->getReg();
1229 SDValue SrcVal = Node->getOperand(Num: 2);
1230 if (DestReg.isVirtual() && SrcVal.isMachineOpcode() &&
1231 SrcVal.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF) {
1232 // Instead building a COPY to that vreg destination, build an
1233 // IMPLICIT_DEF instruction instead.
1234 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Node->getDebugLoc(),
1235 MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg);
1236 break;
1237 }
1238 Register SrcReg;
1239 if (RegisterSDNode *R = dyn_cast<RegisterSDNode>(Val&: SrcVal))
1240 SrcReg = R->getReg();
1241 else
1242 SrcReg = getVR(Op: SrcVal, VRBaseMap);
1243
1244 if (SrcReg == DestReg) // Coalesced away the copy? Ignore.
1245 break;
1246
1247 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Node->getDebugLoc(), MCID: TII->get(Opcode: TargetOpcode::COPY),
1248 DestReg).addReg(RegNo: SrcReg);
1249 break;
1250 }
1251 case ISD::CopyFromReg: {
1252 Register SrcReg = cast<RegisterSDNode>(Val: Node->getOperand(Num: 1))->getReg();
1253 EmitCopyFromReg(Op: SDValue(Node, 0), IsClone, SrcReg, VRBaseMap);
1254 break;
1255 }
1256 case ISD::EH_LABEL:
1257 case ISD::ANNOTATION_LABEL: {
1258 unsigned Opc = (Node->getOpcode() == ISD::EH_LABEL)
1259 ? TargetOpcode::EH_LABEL
1260 : TargetOpcode::ANNOTATION_LABEL;
1261 MCSymbol *S = cast<LabelSDNode>(Val: Node)->getLabel();
1262 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Node->getDebugLoc(),
1263 MCID: TII->get(Opcode: Opc)).addSym(Sym: S);
1264 break;
1265 }
1266
1267 case ISD::LIFETIME_START:
1268 case ISD::LIFETIME_END: {
1269 unsigned TarOp = (Node->getOpcode() == ISD::LIFETIME_START)
1270 ? TargetOpcode::LIFETIME_START
1271 : TargetOpcode::LIFETIME_END;
1272 auto *FI = cast<FrameIndexSDNode>(Val: Node->getOperand(Num: 1));
1273 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Node->getDebugLoc(), MCID: TII->get(Opcode: TarOp))
1274 .addFrameIndex(Idx: FI->getIndex());
1275 break;
1276 }
1277
1278 case ISD::PSEUDO_PROBE: {
1279 unsigned TarOp = TargetOpcode::PSEUDO_PROBE;
1280 auto Guid = cast<PseudoProbeSDNode>(Val: Node)->getGuid();
1281 auto Index = cast<PseudoProbeSDNode>(Val: Node)->getIndex();
1282 auto Attr = cast<PseudoProbeSDNode>(Val: Node)->getAttributes();
1283
1284 BuildMI(BB&: *MBB, I: InsertPos, MIMD: Node->getDebugLoc(), MCID: TII->get(Opcode: TarOp))
1285 .addImm(Val: Guid)
1286 .addImm(Val: Index)
1287 .addImm(Val: (uint8_t)PseudoProbeType::Block)
1288 .addImm(Val: Attr);
1289 break;
1290 }
1291
1292 case ISD::INLINEASM:
1293 case ISD::INLINEASM_BR: {
1294 unsigned NumOps = Node->getNumOperands();
1295 if (Node->getOperand(Num: NumOps-1).getValueType() == MVT::Glue)
1296 --NumOps; // Ignore the glue operand.
1297
1298 // Create the inline asm machine instruction.
1299 unsigned TgtOpc = Node->getOpcode() == ISD::INLINEASM_BR
1300 ? TargetOpcode::INLINEASM_BR
1301 : TargetOpcode::INLINEASM;
1302 MachineInstrBuilder MIB =
1303 BuildMI(MF&: *MF, MIMD: Node->getDebugLoc(), MCID: TII->get(Opcode: TgtOpc));
1304
1305 // Add the asm string as an external symbol operand.
1306 SDValue AsmStrV = Node->getOperand(Num: InlineAsm::Op_AsmString);
1307 const char *AsmStr = cast<ExternalSymbolSDNode>(Val&: AsmStrV)->getSymbol();
1308 MIB.addExternalSymbol(FnName: AsmStr);
1309
1310 // Add the HasSideEffect, isAlignStack, AsmDialect, MayLoad and MayStore
1311 // bits.
1312 int64_t ExtraInfo =
1313 cast<ConstantSDNode>(Val: Node->getOperand(Num: InlineAsm::Op_ExtraInfo))->
1314 getZExtValue();
1315 MIB.addImm(Val: ExtraInfo);
1316
1317 // Remember to operand index of the group flags.
1318 SmallVector<unsigned, 8> GroupIdx;
1319
1320 // Remember registers that are part of early-clobber defs.
1321 SmallVector<Register, 8> ECRegs;
1322
1323 // A glued CopyFromReg may read a clobbered register, e.g. a flag output
1324 // like X86 "={@ccz}" reads EFLAGS defined only by "~{flags}".
1325 SmallVector<Register, 2> GluedUses;
1326 if (Node->getValueType(ResNo: Node->getNumValues() - 1) == MVT::Glue) {
1327 for (SDNode *G = Node->getGluedUser(); G; G = G->getGluedUser()) {
1328 if (G->getOpcode() != ISD::CopyFromReg)
1329 continue;
1330 Register Reg = cast<RegisterSDNode>(Val: G->getOperand(Num: 1))->getReg();
1331 if (Reg.isPhysical())
1332 GluedUses.push_back(Elt: Reg);
1333 }
1334 }
1335
1336 // Add all of the operand registers to the instruction.
1337 for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
1338 unsigned Flags = Node->getConstantOperandVal(Num: i);
1339 const InlineAsm::Flag F(Flags);
1340 const unsigned NumVals = F.getNumOperandRegisters();
1341
1342 GroupIdx.push_back(Elt: MIB->getNumOperands());
1343 MIB.addImm(Val: Flags);
1344 ++i; // Skip the ID value.
1345
1346 switch (F.getKind()) {
1347 case InlineAsm::Kind::RegDef:
1348 for (unsigned j = 0; j != NumVals; ++j, ++i) {
1349 Register Reg = cast<RegisterSDNode>(Val: Node->getOperand(Num: i))->getReg();
1350 // FIXME: Add dead flags for physical and virtual registers defined.
1351 // For now, mark physical register defs as implicit to help fast
1352 // regalloc. This makes inline asm look a lot like calls.
1353 MIB.addReg(RegNo: Reg, Flags: RegState::Define | getImplRegState(B: Reg.isPhysical()));
1354 }
1355 break;
1356 case InlineAsm::Kind::RegDefEarlyClobber:
1357 case InlineAsm::Kind::Clobber:
1358 for (unsigned j = 0; j != NumVals; ++j, ++i) {
1359 Register Reg = cast<RegisterSDNode>(Val: Node->getOperand(Num: i))->getReg();
1360 bool IsDead =
1361 F.isClobberKind() && none_of(Range&: GluedUses, P: [&](Register U) {
1362 return TRI->regsOverlap(RegA: U, RegB: Reg);
1363 });
1364 MIB.addReg(RegNo: Reg, Flags: RegState::Define | RegState::EarlyClobber |
1365 getDeadRegState(B: IsDead) |
1366 getImplRegState(B: Reg.isPhysical()));
1367 ECRegs.push_back(Elt: Reg);
1368 }
1369 break;
1370 case InlineAsm::Kind::RegUse: // Use of register.
1371 case InlineAsm::Kind::Imm: // Immediate.
1372 case InlineAsm::Kind::Mem: // Non-function addressing mode.
1373 // The addressing mode has been selected, just add all of the
1374 // operands to the machine instruction.
1375 for (unsigned j = 0; j != NumVals; ++j, ++i)
1376 AddOperand(MIB, Op: Node->getOperand(Num: i), IIOpNum: 0, II: nullptr, VRBaseMap,
1377 /*IsDebug=*/false, IsClone, IsCloned);
1378
1379 // Manually set isTied bits.
1380 if (F.isRegUseKind()) {
1381 unsigned DefGroup;
1382 if (F.isUseOperandTiedToDef(Idx&: DefGroup)) {
1383 unsigned DefIdx = GroupIdx[DefGroup] + 1;
1384 unsigned UseIdx = GroupIdx.back() + 1;
1385 for (unsigned j = 0; j != NumVals; ++j)
1386 MIB->tieOperands(DefIdx: DefIdx + j, UseIdx: UseIdx + j);
1387 }
1388 }
1389 break;
1390 case InlineAsm::Kind::Func: // Function addressing mode.
1391 for (unsigned j = 0; j != NumVals; ++j, ++i) {
1392 SDValue Op = Node->getOperand(Num: i);
1393 AddOperand(MIB, Op, IIOpNum: 0, II: nullptr, VRBaseMap,
1394 /*IsDebug=*/false, IsClone, IsCloned);
1395
1396 // Adjust Target Flags for function reference.
1397 if (auto *TGA = dyn_cast<GlobalAddressSDNode>(Val&: Op)) {
1398 unsigned NewFlags =
1399 MF->getSubtarget().classifyGlobalFunctionReference(
1400 GV: TGA->getGlobal());
1401 unsigned LastIdx = MIB.getInstr()->getNumOperands() - 1;
1402 MIB.getInstr()->getOperand(i: LastIdx).setTargetFlags(NewFlags);
1403 }
1404 }
1405 }
1406 }
1407
1408 // GCC inline assembly allows input operands to also be early-clobber
1409 // output operands (so long as the operand is written only after it's
1410 // used), but this does not match the semantics of our early-clobber flag.
1411 // If an early-clobber operand register is also an input operand register,
1412 // then remove the early-clobber flag.
1413 for (Register Reg : ECRegs) {
1414 if (MIB->readsRegister(Reg, TRI)) {
1415 MachineOperand *MO =
1416 MIB->findRegisterDefOperand(Reg, TRI, isDead: false, Overlap: false);
1417 assert(MO && "No def operand for clobbered register?");
1418 MO->setIsEarlyClobber(false);
1419 }
1420 }
1421
1422 // Get the mdnode from the asm if it exists and add it to the instruction.
1423 SDValue MDV = Node->getOperand(Num: InlineAsm::Op_MDNode);
1424 const MDNode *MD = cast<MDNodeSDNode>(Val&: MDV)->getMD();
1425 if (MD)
1426 MIB.addMetadata(MD);
1427
1428 // Add rounding control registers as implicit def for inline asm.
1429 if (MF->getFunction().hasFnAttribute(Kind: Attribute::StrictFP)) {
1430 ArrayRef<MCPhysReg> RCRegs = TLI->getRoundingControlRegisters();
1431 for (MCPhysReg Reg : RCRegs)
1432 MIB.addReg(RegNo: Reg, Flags: RegState::ImplicitDefine);
1433 }
1434
1435 MBB->insert(I: InsertPos, MI: MIB);
1436 break;
1437 }
1438 }
1439}
1440
1441/// InstrEmitter - Construct an InstrEmitter and set it to start inserting
1442/// at the given position in the given block.
1443InstrEmitter::InstrEmitter(const TargetMachine &TM, MachineBasicBlock *mbb,
1444 MachineBasicBlock::iterator insertpos)
1445 : MF(mbb->getParent()), MRI(&MF->getRegInfo()),
1446 TII(MF->getSubtarget().getInstrInfo()),
1447 TRI(MF->getSubtarget().getRegisterInfo()),
1448 TLI(MF->getSubtarget().getTargetLowering()), MBB(mbb),
1449 InsertPos(insertpos) {
1450 EmitDebugInstrRefs = mbb->getParent()->useDebugInstrRef();
1451}
1452