1//===--- ScheduleDAGSDNodes.cpp - Implement the ScheduleDAGSDNodes class --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the ScheduleDAG class, which is a base class used by
10// scheduling implementation classes.
11//
12//===----------------------------------------------------------------------===//
13
14#include "ScheduleDAGSDNodes.h"
15#include "InstrEmitter.h"
16#include "SDNodeDbgValue.h"
17#include "SelectionDAGOptions.h"
18#include "llvm/ADT/DenseMap.h"
19#include "llvm/ADT/SmallPtrSet.h"
20#include "llvm/ADT/SmallSet.h"
21#include "llvm/ADT/SmallVector.h"
22#include "llvm/ADT/Statistic.h"
23#include "llvm/CodeGen/MachineInstrBuilder.h"
24#include "llvm/CodeGen/MachineRegisterInfo.h"
25#include "llvm/CodeGen/SelectionDAG.h"
26#include "llvm/CodeGen/TargetInstrInfo.h"
27#include "llvm/CodeGen/TargetLowering.h"
28#include "llvm/CodeGen/TargetRegisterInfo.h"
29#include "llvm/CodeGen/TargetSubtargetInfo.h"
30#include "llvm/Config/llvm-config.h"
31#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
32#include "llvm/MC/MCInstrItineraries.h"
33#include "llvm/Support/CommandLine.h"
34#include "llvm/Support/Debug.h"
35#include "llvm/Support/raw_ostream.h"
36#include "llvm/Target/TargetMachine.h"
37using namespace llvm;
38
39#define DEBUG_TYPE "pre-RA-sched"
40
41STATISTIC(LoadsClustered, "Number of loads clustered together");
42
43ScheduleDAGSDNodes::ScheduleDAGSDNodes(MachineFunction &mf)
44 : ScheduleDAG(mf), InstrItins(mf.getSubtarget().getInstrItineraryData()) {}
45
46/// Run - perform scheduling.
47///
48void ScheduleDAGSDNodes::Run(SelectionDAG *dag, MachineBasicBlock *bb) {
49 BB = bb;
50 DAG = dag;
51
52 // Clear the scheduler's SUnit DAG.
53 ScheduleDAG::clearDAG();
54 Sequence.clear();
55
56 // Invoke the target's selection of scheduler.
57 Schedule();
58}
59
60/// NewSUnit - Creates a new SUnit and return a ptr to it.
61///
62SUnit *ScheduleDAGSDNodes::newSUnit(SDNode *N) {
63#ifndef NDEBUG
64 const SUnit *Addr = nullptr;
65 if (!SUnits.empty())
66 Addr = &SUnits[0];
67#endif
68 SUnits.emplace_back(args&: N, args: (unsigned)SUnits.size());
69 assert((Addr == nullptr || Addr == &SUnits[0]) &&
70 "SUnits std::vector reallocated on the fly!");
71 SUnits.back().OrigNode = &SUnits.back();
72 SUnit *SU = &SUnits.back();
73 const TargetLowering &TLI = DAG->getTargetLoweringInfo();
74 if (!N ||
75 (N->isMachineOpcode() &&
76 N->getMachineOpcode() == TargetOpcode::IMPLICIT_DEF))
77 SU->SchedulingPref = Sched::None;
78 else
79 SU->SchedulingPref = TLI.getSchedulingPreference(N);
80 return SU;
81}
82
83SUnit *ScheduleDAGSDNodes::Clone(SUnit *Old) {
84 SUnit *SU = newSUnit(N: Old->getNode());
85 SU->OrigNode = Old->OrigNode;
86 SU->Latency = Old->Latency;
87 SU->isVRegCycle = Old->isVRegCycle;
88 SU->isCall = Old->isCall;
89 SU->isCallOp = Old->isCallOp;
90 SU->isTwoAddress = Old->isTwoAddress;
91 SU->isCommutable = Old->isCommutable;
92 SU->hasPhysRegDefs = Old->hasPhysRegDefs;
93 SU->hasPhysRegClobbers = Old->hasPhysRegClobbers;
94 SU->isScheduleHigh = Old->isScheduleHigh;
95 SU->isScheduleLow = Old->isScheduleLow;
96 SU->SchedulingPref = Old->SchedulingPref;
97 Old->isCloned = true;
98 return SU;
99}
100
101/// CheckForPhysRegDependency - Check if the dependency between def and use of
102/// a specified operand is a physical register dependency. If so, returns the
103/// register and the cost of copying the register.
104static void CheckForPhysRegDependency(SDNode *Def, SDNode *User, unsigned Op,
105 const TargetRegisterInfo *TRI,
106 const TargetInstrInfo *TII,
107 MCRegister &PhysReg, int &Cost) {
108 if (Op != 2 || User->getOpcode() != ISD::CopyToReg)
109 return;
110
111 Register Reg = cast<RegisterSDNode>(Val: User->getOperand(Num: 1))->getReg();
112 if (Reg.isVirtual())
113 return;
114
115 unsigned ResNo = User->getOperand(Num: 2).getResNo();
116 if (Def->getOpcode() == ISD::CopyFromReg &&
117 cast<RegisterSDNode>(Val: Def->getOperand(Num: 1))->getReg() == Reg) {
118 PhysReg = Reg;
119 } else if (Def->isMachineOpcode()) {
120 const MCInstrDesc &II = TII->get(Opcode: Def->getMachineOpcode());
121 if (ResNo >= II.getNumDefs() && II.hasImplicitDefOfPhysReg(Reg))
122 PhysReg = Reg;
123 }
124
125 if (PhysReg) {
126 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
127 Cost = RC->expensiveOrImpossibleToCopy() ? -1 : RC->getCopyCost();
128 }
129}
130
131// Helper for AddGlue to clone node operands.
132static void CloneNodeWithValues(SDNode *N, SelectionDAG *DAG, ArrayRef<EVT> VTs,
133 SDValue ExtraOper = SDValue()) {
134 SmallVector<SDValue, 8> Ops(N->ops());
135 if (ExtraOper.getNode())
136 Ops.push_back(Elt: ExtraOper);
137
138 SDVTList VTList = DAG->getVTList(VTs);
139 MachineSDNode *MN = dyn_cast<MachineSDNode>(Val: N);
140
141 // Store memory references.
142 SmallVector<MachineMemOperand *, 2> MMOs;
143 if (MN)
144 MMOs.assign(in_start: MN->memoperands_begin(), in_end: MN->memoperands_end());
145
146 DAG->MorphNodeTo(N, Opc: N->getOpcode(), VTs: VTList, Ops);
147
148 // Reset the memory references
149 if (MN)
150 DAG->setNodeMemRefs(N: MN, NewMemRefs: MMOs);
151}
152
153static bool AddGlue(SDNode *N, SDValue Glue, bool AddGlue, SelectionDAG *DAG) {
154 SDNode *GlueDestNode = Glue.getNode();
155
156 // Don't add glue from a node to itself.
157 if (GlueDestNode == N) return false;
158
159 // Don't add a glue operand to something that already uses glue.
160 if (GlueDestNode &&
161 N->getOperand(Num: N->getNumOperands()-1).getValueType() == MVT::Glue) {
162 return false;
163 }
164 // Don't add glue to something that already has a glue value.
165 if (N->getValueType(ResNo: N->getNumValues() - 1) == MVT::Glue) return false;
166
167 SmallVector<EVT, 4> VTs(N->values());
168 if (AddGlue)
169 VTs.push_back(Elt: MVT::Glue);
170
171 CloneNodeWithValues(N, DAG, VTs, ExtraOper: Glue);
172
173 return true;
174}
175
176// Cleanup after unsuccessful AddGlue. Use the standard method of morphing the
177// node even though simply shrinking the value list is sufficient.
178static void RemoveUnusedGlue(SDNode *N, SelectionDAG *DAG) {
179 assert((N->getValueType(N->getNumValues() - 1) == MVT::Glue &&
180 !N->hasAnyUseOfValue(N->getNumValues() - 1)) &&
181 "expected an unused glue value");
182
183 CloneNodeWithValues(N, DAG,
184 VTs: ArrayRef(N->value_begin(), N->getNumValues() - 1));
185}
186
187/// ClusterNeighboringLoads - Force nearby loads together by "gluing" them.
188/// This function finds loads of the same base and different offsets. If the
189/// offsets are not far apart (target specific), it add MVT::Glue inputs and
190/// outputs to ensure they are scheduled together and in order. This
191/// optimization may benefit some targets by improving cache locality.
192void ScheduleDAGSDNodes::ClusterNeighboringLoads(SDNode *Node) {
193 SDValue Chain;
194 unsigned NumOps = Node->getNumOperands();
195 if (Node->getOperand(Num: NumOps-1).getValueType() == MVT::Other)
196 Chain = Node->getOperand(Num: NumOps-1);
197 if (!Chain)
198 return;
199
200 // Skip any load instruction that has a tied input. There may be an additional
201 // dependency requiring a different order than by increasing offsets, and the
202 // added glue may introduce a cycle.
203 auto hasTiedInput = [this](const SDNode *N) {
204 const MCInstrDesc &MCID = TII->get(Opcode: N->getMachineOpcode());
205 for (unsigned I = 0; I != MCID.getNumOperands(); ++I) {
206 if (MCID.getOperandConstraint(OpNum: I, Constraint: MCOI::TIED_TO) != -1)
207 return true;
208 }
209
210 return false;
211 };
212
213 // Look for other loads of the same chain. Find loads that are loading from
214 // the same base pointer and different offsets.
215 SmallPtrSet<SDNode*, 16> Visited;
216 SmallVector<int64_t, 4> Offsets;
217 DenseMap<long long, SDNode*> O2SMap; // Map from offset to SDNode.
218 bool Cluster = false;
219 SDNode *Base = Node;
220
221 if (hasTiedInput(Base))
222 return;
223
224 // This algorithm requires a reasonably low use count before finding a match
225 // to avoid uselessly blowing up compile time in large blocks.
226 unsigned UseCount = 0;
227 for (SDNode::user_iterator I = Chain->user_begin(), E = Chain->user_end();
228 I != E && UseCount < 100; ++I, ++UseCount) {
229 if (I.getUse().getResNo() != Chain.getResNo())
230 continue;
231
232 SDNode *User = *I;
233 if (User == Node || !Visited.insert(Ptr: User).second)
234 continue;
235 int64_t Offset1, Offset2;
236 if (!TII->areLoadsFromSameBasePtr(Load1: Base, Load2: User, Offset1, Offset2) ||
237 Offset1 == Offset2 ||
238 hasTiedInput(User)) {
239 // FIXME: Should be ok if they addresses are identical. But earlier
240 // optimizations really should have eliminated one of the loads.
241 continue;
242 }
243 if (O2SMap.insert(KV: std::make_pair(x&: Offset1, y&: Base)).second)
244 Offsets.push_back(Elt: Offset1);
245 O2SMap.insert(KV: std::make_pair(x&: Offset2, y&: User));
246 Offsets.push_back(Elt: Offset2);
247 if (Offset2 < Offset1)
248 Base = User;
249 Cluster = true;
250 // Reset UseCount to allow more matches.
251 UseCount = 0;
252 }
253
254 if (!Cluster)
255 return;
256
257 // Sort them in increasing order.
258 llvm::sort(C&: Offsets);
259
260 // Check if the loads are close enough.
261 SmallVector<SDNode*, 4> Loads;
262 unsigned NumLoads = 0;
263 int64_t BaseOff = Offsets[0];
264 SDNode *BaseLoad = O2SMap[BaseOff];
265 Loads.push_back(Elt: BaseLoad);
266 for (unsigned i = 1, e = Offsets.size(); i != e; ++i) {
267 int64_t Offset = Offsets[i];
268 SDNode *Load = O2SMap[Offset];
269 if (!TII->shouldScheduleLoadsNear(Load1: BaseLoad, Load2: Load, Offset1: BaseOff, Offset2: Offset,NumLoads))
270 break; // Stop right here. Ignore loads that are further away.
271 Loads.push_back(Elt: Load);
272 ++NumLoads;
273 }
274
275 if (NumLoads == 0)
276 return;
277
278 // Cluster loads by adding MVT::Glue outputs and inputs. This also
279 // ensure they are scheduled in order of increasing addresses.
280 SDNode *Lead = Loads[0];
281 SDValue InGlue;
282 if (AddGlue(N: Lead, Glue: InGlue, AddGlue: true, DAG))
283 InGlue = SDValue(Lead, Lead->getNumValues() - 1);
284 for (unsigned I = 1, E = Loads.size(); I != E; ++I) {
285 bool OutGlue = I < E - 1;
286 SDNode *Load = Loads[I];
287
288 // If AddGlue fails, we could leave an unsused glue value. This should not
289 // cause any
290 if (AddGlue(N: Load, Glue: InGlue, AddGlue: OutGlue, DAG)) {
291 if (OutGlue)
292 InGlue = SDValue(Load, Load->getNumValues() - 1);
293
294 ++LoadsClustered;
295 }
296 else if (!OutGlue && InGlue.getNode())
297 RemoveUnusedGlue(N: InGlue.getNode(), DAG);
298 }
299}
300
301/// ClusterNodes - Cluster certain nodes which should be scheduled together.
302///
303void ScheduleDAGSDNodes::ClusterNodes() {
304 for (SDNode &NI : DAG->allnodes()) {
305 SDNode *Node = &NI;
306 if (!Node || !Node->isMachineOpcode())
307 continue;
308
309 unsigned Opc = Node->getMachineOpcode();
310 const MCInstrDesc &MCID = TII->get(Opcode: Opc);
311 if (MCID.mayLoad())
312 // Cluster loads from "near" addresses into combined SUnits.
313 ClusterNeighboringLoads(Node);
314 }
315}
316
317void ScheduleDAGSDNodes::BuildSchedUnits() {
318 // During scheduling, the NodeId field of SDNode is used to map SDNodes
319 // to their associated SUnits by holding SUnits table indices. A value
320 // of -1 means the SDNode does not yet have an associated SUnit.
321 unsigned NumNodes = 0;
322 for (SDNode &NI : DAG->allnodes()) {
323 NI.setNodeId(-1);
324 NI.setSchedulerWorklistVisited(false);
325 ++NumNodes;
326 }
327
328 // Reserve entries in the vector for each of the SUnits we are creating. This
329 // ensure that reallocation of the vector won't happen, so SUnit*'s won't get
330 // invalidated.
331 // FIXME: Multiply by 2 because we may clone nodes during scheduling.
332 // This is a temporary workaround.
333 SUnits.reserve(n: NumNodes * 2);
334
335 // Add all nodes in depth first order.
336 SmallVector<SDNode *, 64> Worklist;
337 Worklist.push_back(Elt: DAG->getRoot().getNode());
338 DAG->getRoot().getNode()->setSchedulerWorklistVisited(true);
339
340 SmallVector<SUnit*, 8> CallSUnits;
341 while (!Worklist.empty()) {
342 SDNode *NI = Worklist.pop_back_val();
343
344 // Add all operands to the worklist unless they've already been added.
345 for (const SDValue &Op : NI->op_values()) {
346 if (Op.getNode()->getSchedulerWorklistVisited())
347 continue;
348 Op.getNode()->setSchedulerWorklistVisited(true);
349 Worklist.push_back(Elt: Op.getNode());
350 }
351
352 if (isPassiveNode(Node: NI)) // Leaf node, e.g. a TargetImmediate.
353 continue;
354
355 // If this node has already been processed, stop now.
356 if (NI->getNodeId() != -1) continue;
357
358 SUnit *NodeSUnit = newSUnit(N: NI);
359
360 // See if anything is glued to this node, if so, add them to glued
361 // nodes. Nodes can have at most one glue input and one glue output. Glue
362 // is required to be the last operand and result of a node.
363
364 // Scan up to find glued preds.
365 SDNode *N = NI;
366 while (N->getNumOperands() &&
367 N->getOperand(Num: N->getNumOperands()-1).getValueType() == MVT::Glue) {
368 N = N->getOperand(Num: N->getNumOperands()-1).getNode();
369 assert(N->getNodeId() == -1 && "Node already inserted!");
370 N->setNodeId(NodeSUnit->NodeNum);
371 if (N->isMachineOpcode() && TII->get(Opcode: N->getMachineOpcode()).isCall())
372 NodeSUnit->isCall = true;
373 }
374
375 // Scan down to find any glued succs.
376 N = NI;
377 while (N->getValueType(ResNo: N->getNumValues()-1) == MVT::Glue) {
378 SDValue GlueVal(N, N->getNumValues()-1);
379
380 // There are either zero or one users of the Glue result.
381 bool HasGlueUse = false;
382 for (SDNode *U : N->users())
383 if (GlueVal.isOperandOf(N: U)) {
384 HasGlueUse = true;
385 assert(N->getNodeId() == -1 && "Node already inserted!");
386 N->setNodeId(NodeSUnit->NodeNum);
387 N = U;
388 if (N->isMachineOpcode() && TII->get(Opcode: N->getMachineOpcode()).isCall())
389 NodeSUnit->isCall = true;
390 break;
391 }
392 if (!HasGlueUse) break;
393 }
394
395 if (NodeSUnit->isCall)
396 CallSUnits.push_back(Elt: NodeSUnit);
397
398 // Schedule zero-latency TokenFactor below any nodes that may increase the
399 // schedule height. Otherwise, ancestors of the TokenFactor may appear to
400 // have false stalls.
401 if (NI->getOpcode() == ISD::TokenFactor)
402 NodeSUnit->isScheduleLow = true;
403
404 // If there are glue operands involved, N is now the bottom-most node
405 // of the sequence of nodes that are glued together.
406 // Update the SUnit.
407 NodeSUnit->setNode(N);
408 assert(N->getNodeId() == -1 && "Node already inserted!");
409 N->setNodeId(NodeSUnit->NodeNum);
410
411 // Compute NumRegDefsLeft. This must be done before AddSchedEdges.
412 InitNumRegDefsLeft(SU: NodeSUnit);
413
414 // Assign the Latency field of NodeSUnit using target-provided information.
415 computeLatency(SU: NodeSUnit);
416 }
417
418 // Find all call operands.
419 while (!CallSUnits.empty()) {
420 SUnit *SU = CallSUnits.pop_back_val();
421 for (const SDNode *SUNode = SU->getNode(); SUNode;
422 SUNode = SUNode->getGluedNode()) {
423 if (SUNode->getOpcode() != ISD::CopyToReg)
424 continue;
425 SDNode *SrcN = SUNode->getOperand(Num: 2).getNode();
426 if (isPassiveNode(Node: SrcN)) continue; // Not scheduled.
427 SUnit *SrcSU = &SUnits[SrcN->getNodeId()];
428 SrcSU->isCallOp = true;
429 }
430 }
431}
432
433void ScheduleDAGSDNodes::AddSchedEdges() {
434 const TargetSubtargetInfo &ST = MF.getSubtarget();
435
436 // Check to see if the scheduler cares about latencies.
437 bool UnitLatencies = forceUnitLatencies();
438
439 // Pass 2: add the preds, succs, etc.
440 for (SUnit &SU : SUnits) {
441 SDNode *MainNode = SU.getNode();
442
443 if (MainNode->isMachineOpcode()) {
444 unsigned Opc = MainNode->getMachineOpcode();
445 const MCInstrDesc &MCID = TII->get(Opcode: Opc);
446 for (unsigned i = 0; i != MCID.getNumOperands(); ++i) {
447 if (MCID.getOperandConstraint(OpNum: i, Constraint: MCOI::TIED_TO) != -1) {
448 SU.isTwoAddress = true;
449 break;
450 }
451 }
452 if (MCID.isCommutable())
453 SU.isCommutable = true;
454 }
455
456 // Find all predecessors and successors of the group.
457 for (SDNode *N = SU.getNode(); N; N = N->getGluedNode()) {
458 if (N->isMachineOpcode() &&
459 !TII->get(Opcode: N->getMachineOpcode()).implicit_defs().empty()) {
460 const MCInstrDesc &MCID = TII->get(Opcode: N->getMachineOpcode());
461 SU.hasPhysRegClobbers = true;
462 // Results beyond the explicit defs are physical register defs, unless
463 // the target models variadic results as virtual registers. This must
464 // match InstrEmitter::EmitMachineNode.
465 bool HasVRegVariadicDefs = !MF.getTarget().usesPhysRegsForValues() &&
466 MCID.isVariadic() &&
467 MCID.variadicOpsAreDefs();
468 unsigned NumUsed = InstrEmitter::CountResults(Node: N);
469 while (NumUsed != 0 && !N->hasAnyUseOfValue(Value: NumUsed - 1))
470 --NumUsed; // Skip over unused values at the end.
471 if (!HasVRegVariadicDefs && NumUsed > MCID.getNumDefs())
472 SU.hasPhysRegDefs = true;
473 }
474
475 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
476 SDNode *OpN = N->getOperand(Num: i).getNode();
477 unsigned DefIdx = N->getOperand(Num: i).getResNo();
478 if (isPassiveNode(Node: OpN)) continue; // Not scheduled.
479 SUnit *OpSU = &SUnits[OpN->getNodeId()];
480 assert(OpSU && "Node has no SUnit!");
481 if (OpSU == &SU)
482 continue; // In the same group.
483
484 EVT OpVT = N->getOperand(Num: i).getValueType();
485 assert(OpVT != MVT::Glue && "Glued nodes should be in same sunit!");
486 bool isChain = OpVT == MVT::Other;
487
488 MCRegister PhysReg;
489 int Cost = 1;
490 // Determine if this is a physical register dependency.
491 CheckForPhysRegDependency(Def: OpN, User: N, Op: i, TRI, TII, PhysReg, Cost);
492 assert((!PhysReg || !isChain) && "Chain dependence via physreg data?");
493 // FIXME: See ScheduleDAGSDNodes::EmitCopyFromReg. For now, scheduler
494 // emits a copy from the physical register to a virtual register unless
495 // it requires a cross class copy (cost < 0). That means we are only
496 // treating "expensive to copy" register dependency as physical register
497 // dependency. This may change in the future though.
498 if (Cost >= 0 && !StressSched)
499 PhysReg = MCRegister();
500
501 // If this is a ctrl dep, latency is 1.
502 unsigned OpLatency = isChain ? 1 : OpSU->Latency;
503 // Special-case TokenFactor chains as zero-latency.
504 if(isChain && OpN->getOpcode() == ISD::TokenFactor)
505 OpLatency = 0;
506
507 SDep Dep = isChain ? SDep(OpSU, SDep::Barrier)
508 : SDep(OpSU, SDep::Data, PhysReg);
509 Dep.setLatency(OpLatency);
510 if (!isChain && !UnitLatencies) {
511 computeOperandLatency(Def: OpN, Use: N, OpIdx: i, dep&: Dep);
512 ST.adjustSchedDependency(Def: OpSU, DefOpIdx: DefIdx, Use: &SU, UseOpIdx: i, Dep, SchedModel: nullptr);
513 }
514
515 if (!SU.addPred(D: Dep) && !Dep.isCtrl() && OpSU->NumRegDefsLeft > 1) {
516 // Multiple register uses are combined in the same SUnit. For example,
517 // we could have a set of glued nodes with all their defs consumed by
518 // another set of glued nodes. Register pressure tracking sees this as
519 // a single use, so to keep pressure balanced we reduce the defs.
520 //
521 // We can't tell (without more book-keeping) if this results from
522 // glued nodes or duplicate operands. As long as we don't reduce
523 // NumRegDefsLeft to zero, we handle the common cases well.
524 --OpSU->NumRegDefsLeft;
525 }
526 }
527 }
528 }
529}
530
531/// BuildSchedGraph - Build the SUnit graph from the selection dag that we
532/// are input. This SUnit graph is similar to the SelectionDAG, but
533/// excludes nodes that aren't interesting to scheduling, and represents
534/// glued together nodes with a single SUnit.
535void ScheduleDAGSDNodes::BuildSchedGraph() {
536 // Cluster certain nodes which should be scheduled together.
537 ClusterNodes();
538 // Populate the SUnits array.
539 BuildSchedUnits();
540 // Compute all the scheduling dependencies between nodes.
541 AddSchedEdges();
542}
543
544// Initialize NumNodeDefs for the current Node's opcode.
545void ScheduleDAGSDNodes::RegDefIter::InitNodeNumDefs() {
546 // Check for phys reg copy.
547 if (!Node)
548 return;
549
550 if (!Node->isMachineOpcode()) {
551 if (Node->getOpcode() == ISD::CopyFromReg)
552 NodeNumDefs = 1;
553 else
554 NodeNumDefs = 0;
555 return;
556 }
557 unsigned POpc = Node->getMachineOpcode();
558 if (POpc == TargetOpcode::IMPLICIT_DEF) {
559 // No register need be allocated for this.
560 NodeNumDefs = 0;
561 return;
562 }
563 if (POpc == TargetOpcode::PATCHPOINT &&
564 Node->getValueType(ResNo: 0) == MVT::Other) {
565 // PATCHPOINT is defined to have one result, but it might really have none
566 // if we're not using CallingConv::AnyReg. Don't mistake the chain for a
567 // real definition.
568 NodeNumDefs = 0;
569 return;
570 }
571 unsigned NRegDefs = SchedDAG->TII->get(Opcode: Node->getMachineOpcode()).getNumDefs();
572 // Some instructions define regs that are not represented in the selection DAG
573 // (e.g. unused flags). See tMOVi8. Make sure we don't access past NumValues.
574 NodeNumDefs = std::min(a: Node->getNumValues(), b: NRegDefs);
575 DefIdx = 0;
576}
577
578// Construct a RegDefIter for this SUnit and find the first valid value.
579ScheduleDAGSDNodes::RegDefIter::RegDefIter(const SUnit *SU,
580 const ScheduleDAGSDNodes *SD)
581 : SchedDAG(SD), Node(SU->getNode()) {
582 InitNodeNumDefs();
583 Advance();
584}
585
586// Advance to the next valid value defined by the SUnit.
587void ScheduleDAGSDNodes::RegDefIter::Advance() {
588 for (;Node;) { // Visit all glued nodes.
589 for (;DefIdx < NodeNumDefs; ++DefIdx) {
590 if (!Node->hasAnyUseOfValue(Value: DefIdx))
591 continue;
592 ValueType = Node->getSimpleValueType(ResNo: DefIdx);
593 ++DefIdx;
594 return; // Found a normal regdef.
595 }
596 Node = Node->getGluedNode();
597 if (!Node) {
598 return; // No values left to visit.
599 }
600 InitNodeNumDefs();
601 }
602}
603
604void ScheduleDAGSDNodes::InitNumRegDefsLeft(SUnit *SU) {
605 assert(SU->NumRegDefsLeft == 0 && "expect a new node");
606 for (RegDefIter I(SU, this); I.IsValid(); I.Advance()) {
607 assert(SU->NumRegDefsLeft < USHRT_MAX && "overflow is ok but unexpected");
608 ++SU->NumRegDefsLeft;
609 }
610}
611
612void ScheduleDAGSDNodes::computeLatency(SUnit *SU) {
613 SDNode *N = SU->getNode();
614
615 // TokenFactor operands are considered zero latency, and some schedulers
616 // (e.g. Top-Down list) may rely on the fact that operand latency is nonzero
617 // whenever node latency is nonzero.
618 if (N && N->getOpcode() == ISD::TokenFactor) {
619 SU->Latency = 0;
620 return;
621 }
622
623 // Check to see if the scheduler cares about latencies.
624 if (forceUnitLatencies()) {
625 SU->Latency = 1;
626 return;
627 }
628
629 if (!InstrItins || InstrItins->isEmpty()) {
630 if (N && N->isMachineOpcode() &&
631 TII->isHighLatencyDef(opc: N->getMachineOpcode()))
632 SU->Latency = DAG->getSelectionDAGOptions().sched_high_latency_cycles;
633 else
634 SU->Latency = 1;
635 return;
636 }
637
638 // Compute the latency for the node. We use the sum of the latencies for
639 // all nodes glued together into this SUnit.
640 SU->Latency = 0;
641 for (SDNode *N = SU->getNode(); N; N = N->getGluedNode())
642 if (N->isMachineOpcode())
643 SU->Latency += TII->getInstrLatency(ItinData: InstrItins, Node: N);
644}
645
646void ScheduleDAGSDNodes::computeOperandLatency(SDNode *Def, SDNode *Use,
647 unsigned OpIdx, SDep& dep) const{
648 // Check to see if the scheduler cares about latencies.
649 if (forceUnitLatencies())
650 return;
651
652 if (dep.getKind() != SDep::Data)
653 return;
654
655 unsigned DefIdx = Use->getOperand(Num: OpIdx).getResNo();
656 if (Use->isMachineOpcode())
657 // Adjust the use operand index by num of defs.
658 OpIdx += TII->get(Opcode: Use->getMachineOpcode()).getNumDefs();
659 std::optional<unsigned> Latency =
660 TII->getOperandLatency(ItinData: InstrItins, DefNode: Def, DefIdx, UseNode: Use, UseIdx: OpIdx);
661 if (Latency > 1U && Use->getOpcode() == ISD::CopyToReg &&
662 !BB->succ_empty()) {
663 Register Reg = cast<RegisterSDNode>(Val: Use->getOperand(Num: 1))->getReg();
664 if (Reg.isVirtual())
665 // This copy is a liveout value. It is likely coalesced, so reduce the
666 // latency so not to penalize the def.
667 // FIXME: need target specific adjustment here?
668 Latency = *Latency - 1;
669 }
670 if (Latency)
671 dep.setLatency(*Latency);
672}
673
674void ScheduleDAGSDNodes::dumpNode(const SUnit &SU) const {
675#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
676 dumpNodeName(SU);
677 dbgs() << ": ";
678
679 if (!SU.getNode()) {
680 dbgs() << "PHYS REG COPY\n";
681 return;
682 }
683
684 SU.getNode()->dump(DAG);
685 dbgs() << "\n";
686 SmallVector<SDNode *, 4> GluedNodes;
687 for (SDNode *N = SU.getNode()->getGluedNode(); N; N = N->getGluedNode())
688 GluedNodes.push_back(N);
689 while (!GluedNodes.empty()) {
690 dbgs() << " ";
691 GluedNodes.back()->dump(DAG);
692 dbgs() << "\n";
693 GluedNodes.pop_back();
694 }
695#endif
696}
697
698void ScheduleDAGSDNodes::dump() const {
699#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
700 if (EntrySU.getNode() != nullptr)
701 dumpNodeAll(EntrySU);
702 for (const SUnit &SU : SUnits)
703 dumpNodeAll(SU);
704 if (ExitSU.getNode() != nullptr)
705 dumpNodeAll(ExitSU);
706#endif
707}
708
709#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
710void ScheduleDAGSDNodes::dumpSchedule() const {
711 for (const SUnit *SU : Sequence) {
712 if (SU)
713 dumpNode(*SU);
714 else
715 dbgs() << "**** NOOP ****\n";
716 }
717}
718#endif
719
720#ifndef NDEBUG
721/// VerifyScheduledSequence - Verify that all SUnits were scheduled and that
722/// their state is consistent with the nodes listed in Sequence.
723///
724void ScheduleDAGSDNodes::VerifyScheduledSequence(bool isBottomUp) {
725 unsigned ScheduledNodes = ScheduleDAG::VerifyScheduledDAG(isBottomUp);
726 unsigned Noops = llvm::count(Sequence, nullptr);
727 assert(Sequence.size() - Noops == ScheduledNodes &&
728 "The number of nodes scheduled doesn't match the expected number!");
729}
730#endif // NDEBUG
731
732/// ProcessSDDbgValues - Process SDDbgValues associated with this node.
733static void
734ProcessSDDbgValues(SDNode *N, SelectionDAG *DAG, InstrEmitter &Emitter,
735 SmallVectorImpl<std::pair<unsigned, MachineInstr*> > &Orders,
736 InstrEmitter::VRBaseMapType &VRBaseMap, unsigned Order) {
737 if (!N->getHasDebugValue())
738 return;
739
740 /// Returns true if \p DV has any VReg operand locations which don't exist in
741 /// VRBaseMap.
742 auto HasUnknownVReg = [&VRBaseMap](SDDbgValue *DV) {
743 for (const SDDbgOperand &L : DV->getLocationOps()) {
744 if (L.getKind() == SDDbgOperand::SDNODE &&
745 VRBaseMap.count(Val: {L.getSDNode(), L.getResNo()}) == 0)
746 return true;
747 }
748 return false;
749 };
750
751 // Opportunistically insert immediate dbg_value uses, i.e. those with the same
752 // source order number as N.
753 MachineBasicBlock *BB = Emitter.getBlock();
754 MachineBasicBlock::iterator InsertPos = Emitter.getInsertPos();
755 for (auto *DV : DAG->GetDbgValues(SD: N)) {
756 if (DV->isEmitted())
757 continue;
758 unsigned DVOrder = DV->getOrder();
759 if (Order != 0 && DVOrder != Order)
760 continue;
761 // If DV has any VReg location operands which haven't been mapped then
762 // either that node is no longer available or we just haven't visited the
763 // node yet. In the former case we should emit an undef dbg_value, but we
764 // can do it later. And for the latter we'll want to wait until all
765 // dependent nodes have been visited.
766 if (!DV->isInvalidated() && HasUnknownVReg(DV))
767 continue;
768 MachineInstr *DbgMI = Emitter.EmitDbgValue(SD: DV, VRBaseMap);
769 if (!DbgMI)
770 continue;
771 Orders.push_back(Elt: {DVOrder, DbgMI});
772 BB->insert(I: InsertPos, MI: DbgMI);
773 }
774}
775
776// ProcessSourceNode - Process nodes with source order numbers. These are added
777// to a vector which EmitSchedule uses to determine how to insert dbg_value
778// instructions in the right order.
779static void
780ProcessSourceNode(SDNode *N, SelectionDAG *DAG, InstrEmitter &Emitter,
781 InstrEmitter::VRBaseMapType &VRBaseMap,
782 SmallVectorImpl<std::pair<unsigned, MachineInstr *>> &Orders,
783 SmallSet<Register, 8> &Seen, MachineInstr *NewInsn) {
784 unsigned Order = N->getIROrder();
785 if (!Order || Seen.count(V: Order)) {
786 // Process any valid SDDbgValues even if node does not have any order
787 // assigned.
788 ProcessSDDbgValues(N, DAG, Emitter, Orders, VRBaseMap, Order: 0);
789 return;
790 }
791
792 // If a new instruction was generated for this Order number, record it.
793 // Otherwise, leave this order number unseen: we will either find later
794 // instructions for it, or leave it unseen if there were no instructions at
795 // all.
796 if (NewInsn) {
797 Seen.insert(V: Order);
798 Orders.push_back(Elt: {Order, NewInsn});
799 }
800
801 // Even if no instruction was generated, a Value may have become defined via
802 // earlier nodes. Try to process them now.
803 ProcessSDDbgValues(N, DAG, Emitter, Orders, VRBaseMap, Order);
804}
805
806void ScheduleDAGSDNodes::
807EmitPhysRegCopy(SUnit *SU, SmallDenseMap<SUnit *, Register, 16> &VRBaseMap,
808 MachineBasicBlock::iterator InsertPos) {
809 for (const SDep &Pred : SU->Preds) {
810 if (Pred.isCtrl())
811 continue; // ignore chain preds
812 if (Pred.getSUnit()->CopyDstRC) {
813 // Copy to physical register.
814 auto VRI = VRBaseMap.find(Val: Pred.getSUnit());
815 assert(VRI != VRBaseMap.end() && "Node emitted out of order - late");
816 // Find the destination physical register.
817 Register Reg;
818 for (const SDep &Succ : SU->Succs) {
819 if (Succ.isCtrl())
820 continue; // ignore chain preds
821 if (Succ.getReg()) {
822 Reg = Succ.getReg();
823 break;
824 }
825 }
826 BuildMI(BB&: *BB, I: InsertPos, MIMD: DebugLoc(), MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: Reg)
827 .addReg(RegNo: VRI->second);
828 } else {
829 // Copy from physical register.
830 assert(Pred.getReg() && "Unknown physical register!");
831 Register VRBase = MRI.createVirtualRegister(RegClass: SU->CopyDstRC);
832 bool isNew = VRBaseMap.insert(KV: std::make_pair(x&: SU, y&: VRBase)).second;
833 (void)isNew; // Silence compiler warning.
834 assert(isNew && "Node emitted out of order - early");
835 BuildMI(BB&: *BB, I: InsertPos, MIMD: DebugLoc(), MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: VRBase)
836 .addReg(RegNo: Pred.getReg());
837 }
838 break;
839 }
840}
841
842/// Mark physreg defs dead on \p MI, emitted for a cloned node \p SU. All copies
843/// share one SDNode, so InstrEmitter can't tell which copy lost its users. A
844/// live def of an expensive-to-copy physreg is carried on a scheduler edge by
845/// some copy, so it is dead if \p SU has no such edge.
846static void markClonedNodeDeadDefs(const SUnit &SU, MachineInstr &MI,
847 const TargetRegisterInfo *TRI) {
848 for (MachineOperand &MO : MI.implicit_operands()) {
849 if (!MO.isReg() || !MO.isDef() || MO.isDead())
850 continue;
851 Register Reg = MO.getReg();
852 if (!TRI->getMinimalPhysRegClass(Reg)->expensiveOrImpossibleToCopy())
853 continue;
854 if (none_of(Range: SU.Succs, P: [=](const SDep &Succ) {
855 return Succ.isAssignedRegDep() && Succ.getReg() == Reg;
856 }))
857 MO.setIsDead();
858 }
859}
860
861/// EmitSchedule - Emit the machine code in scheduled order. Return the new
862/// InsertPos and MachineBasicBlock that contains this insertion
863/// point. ScheduleDAGSDNodes holds a BB pointer for convenience, but this does
864/// not necessarily refer to returned BB. The emitter may split blocks.
865MachineBasicBlock *ScheduleDAGSDNodes::
866EmitSchedule(MachineBasicBlock::iterator &InsertPos) {
867 InstrEmitter Emitter(DAG->getTarget(), BB, InsertPos);
868 InstrEmitter::VRBaseMapType VRBaseMap;
869 SmallDenseMap<SUnit *, Register, 16> CopyVRBaseMap;
870 SmallVector<std::pair<unsigned, MachineInstr*>, 32> Orders;
871 SmallSet<Register, 8> Seen;
872 bool HasDbg = DAG->hasDebugValues();
873
874 // Emit a node, and determine where its first instruction is for debuginfo.
875 // Zero, one, or multiple instructions can be created when emitting a node.
876 auto EmitNode =
877 [&](SDNode *Node, bool IsClone, bool IsCloned,
878 InstrEmitter::VRBaseMapType &VRBaseMap) -> MachineInstr * {
879 // Fetch instruction prior to this, or end() if nonexistant.
880 auto GetPrevInsn = [&](MachineBasicBlock::iterator I) {
881 if (I == BB->begin())
882 return BB->end();
883 else
884 return std::prev(x: Emitter.getInsertPos());
885 };
886
887 MachineBasicBlock::iterator Before = GetPrevInsn(Emitter.getInsertPos());
888 Emitter.EmitNode(Node, IsClone, IsCloned, VRBaseMap);
889 MachineBasicBlock::iterator After = GetPrevInsn(Emitter.getInsertPos());
890
891 // If the iterator did not change, no instructions were inserted.
892 if (Before == After)
893 return nullptr;
894
895 MachineInstr *MI;
896 if (Before == BB->end()) {
897 // There were no prior instructions; the new ones must start at the
898 // beginning of the block.
899 MI = &Emitter.getBlock()->instr_front();
900 } else {
901 // Return first instruction after the pre-existing instructions.
902 MI = &*std::next(x: Before);
903 }
904
905 if (MI->isCandidateForAdditionalCallInfo()) {
906 if (DAG->getTarget().Options.EmitCallSiteInfo ||
907 DAG->getTarget().Options.EmitCallGraphSection)
908 MF.addCallSiteInfo(CallI: MI, CallInfo: DAG->getCallSiteInfo(Node));
909
910 if (auto CalledGlobal = DAG->getCalledGlobal(Node))
911 if (CalledGlobal->Callee)
912 MF.addCalledGlobal(MI, Details: *CalledGlobal);
913 }
914
915 if (DAG->getNoMergeSiteInfo(Node)) {
916 MI->setFlag(MachineInstr::MIFlag::NoMerge);
917 }
918
919 if (MDNode *MD = DAG->getPCSections(Node))
920 MI->setPCSections(MF, MD);
921
922 // Set MMRAs on _all_ added instructions.
923 if (MDNode *MMRA = DAG->getMMRAMetadata(Node)) {
924 for (MachineBasicBlock::iterator It = MI->getIterator(),
925 End = std::next(x: After);
926 It != End; ++It)
927 It->setMMRAMetadata(MF, MMRAs: MMRA);
928 }
929
930 return MI;
931 };
932
933 // If this is the first BB, emit byval parameter dbg_value's.
934 if (HasDbg && BB->getParent()->begin() == MachineFunction::iterator(BB)) {
935 SDDbgInfo::DbgIterator PDI = DAG->ByvalParmDbgBegin();
936 SDDbgInfo::DbgIterator PDE = DAG->ByvalParmDbgEnd();
937 for (; PDI != PDE; ++PDI) {
938 MachineInstr *DbgMI= Emitter.EmitDbgValue(SD: *PDI, VRBaseMap);
939 if (DbgMI) {
940 BB->insert(I: InsertPos, MI: DbgMI);
941 // We re-emit the dbg_value closer to its use, too, after instructions
942 // are emitted to the BB.
943 (*PDI)->clearIsEmitted();
944 }
945 }
946 }
947
948 for (SUnit *SU : Sequence) {
949 if (!SU) {
950 // Null SUnit* is a noop.
951 TII->insertNoop(MBB&: *Emitter.getBlock(), MI: InsertPos);
952 continue;
953 }
954
955 // For pre-regalloc scheduling, create instructions corresponding to the
956 // SDNode and any glued SDNodes and append them to the block.
957 if (!SU->getNode()) {
958 // Emit a copy.
959 EmitPhysRegCopy(SU, VRBaseMap&: CopyVRBaseMap, InsertPos);
960 continue;
961 }
962
963 SmallVector<SDNode *, 4> GluedNodes;
964 for (SDNode *N = SU->getNode()->getGluedNode(); N; N = N->getGluedNode())
965 GluedNodes.push_back(Elt: N);
966 while (!GluedNodes.empty()) {
967 SDNode *N = GluedNodes.back();
968 auto NewInsn = EmitNode(N, SU->OrigNode != SU, SU->isCloned, VRBaseMap);
969 // Remember the source order of the inserted instruction.
970 if (HasDbg)
971 ProcessSourceNode(N, DAG, Emitter, VRBaseMap, Orders, Seen, NewInsn);
972
973 if (MDNode *MD = DAG->getHeapAllocSite(Node: N))
974 if (NewInsn && NewInsn->isCall())
975 NewInsn->setHeapAllocMarker(MF, MD);
976
977 GluedNodes.pop_back();
978 }
979 auto NewInsn =
980 EmitNode(SU->getNode(), SU->OrigNode != SU, SU->isCloned, VRBaseMap);
981 // Remember the source order of the inserted instruction.
982 if (HasDbg)
983 ProcessSourceNode(N: SU->getNode(), DAG, Emitter, VRBaseMap, Orders, Seen,
984 NewInsn);
985
986 if (MDNode *MD = DAG->getHeapAllocSite(Node: SU->getNode())) {
987 if (NewInsn && NewInsn->isCall())
988 NewInsn->setHeapAllocMarker(MF, MD);
989 }
990
991 if (NewInsn && SU->OrigNode->isCloned)
992 markClonedNodeDeadDefs(SU: *SU, MI&: *NewInsn, TRI);
993 }
994
995 // Insert all the dbg_values which have not already been inserted in source
996 // order sequence.
997 if (HasDbg) {
998 MachineBasicBlock::iterator BBBegin = BB->getFirstNonPHI();
999
1000 // Sort the source order instructions and use the order to insert debug
1001 // values. Use stable_sort so that DBG_VALUEs are inserted in the same order
1002 // regardless of the host's implementation fo std::sort.
1003 llvm::stable_sort(Range&: Orders, C: less_first());
1004 std::stable_sort(first: DAG->DbgBegin(), last: DAG->DbgEnd(),
1005 comp: [](const SDDbgValue *LHS, const SDDbgValue *RHS) {
1006 return LHS->getOrder() < RHS->getOrder();
1007 });
1008
1009 SDDbgInfo::DbgIterator DI = DAG->DbgBegin();
1010 SDDbgInfo::DbgIterator DE = DAG->DbgEnd();
1011 // Now emit the rest according to source order.
1012 unsigned LastOrder = 0;
1013 for (unsigned i = 0, e = Orders.size(); i != e && DI != DE; ++i) {
1014 unsigned Order = Orders[i].first;
1015 MachineInstr *MI = Orders[i].second;
1016 // Insert all SDDbgValue's whose order(s) are before "Order".
1017 assert(MI);
1018 for (; DI != DE; ++DI) {
1019 if ((*DI)->getOrder() < LastOrder || (*DI)->getOrder() >= Order)
1020 break;
1021 if ((*DI)->isEmitted())
1022 continue;
1023
1024 MachineInstr *DbgMI = Emitter.EmitDbgValue(SD: *DI, VRBaseMap);
1025 if (DbgMI) {
1026 if (!LastOrder)
1027 // Insert to start of the BB (after PHIs).
1028 BB->insert(I: BBBegin, MI: DbgMI);
1029 else {
1030 // Insert at the instruction, which may be in a different
1031 // block, if the block was split by a custom inserter.
1032 MachineBasicBlock::iterator Pos = MI;
1033 MI->getParent()->insert(I: Pos, MI: DbgMI);
1034 }
1035 }
1036 }
1037 LastOrder = Order;
1038 }
1039 // Add trailing DbgValue's before the terminator. FIXME: May want to add
1040 // some of them before one or more conditional branches?
1041 SmallVector<MachineInstr*, 8> DbgMIs;
1042 for (; DI != DE; ++DI) {
1043 if ((*DI)->isEmitted())
1044 continue;
1045 assert((*DI)->getOrder() >= LastOrder &&
1046 "emitting DBG_VALUE out of order");
1047 if (MachineInstr *DbgMI = Emitter.EmitDbgValue(SD: *DI, VRBaseMap))
1048 DbgMIs.push_back(Elt: DbgMI);
1049 }
1050
1051 MachineBasicBlock *InsertBB = Emitter.getBlock();
1052 MachineBasicBlock::iterator Pos = InsertBB->getFirstTerminator();
1053 InsertBB->insert(I: Pos, S: DbgMIs.begin(), E: DbgMIs.end());
1054
1055 SDDbgInfo::DbgLabelIterator DLI = DAG->DbgLabelBegin();
1056 SDDbgInfo::DbgLabelIterator DLE = DAG->DbgLabelEnd();
1057 // Now emit the rest according to source order.
1058 LastOrder = 0;
1059 for (const auto &InstrOrder : Orders) {
1060 unsigned Order = InstrOrder.first;
1061 MachineInstr *MI = InstrOrder.second;
1062 if (!MI)
1063 continue;
1064
1065 // Insert all SDDbgLabel's whose order(s) are before "Order".
1066 for (; DLI != DLE &&
1067 (*DLI)->getOrder() >= LastOrder && (*DLI)->getOrder() < Order;
1068 ++DLI) {
1069 MachineInstr *DbgMI = Emitter.EmitDbgLabel(SD: *DLI);
1070 if (DbgMI) {
1071 if (!LastOrder)
1072 // Insert to start of the BB (after PHIs).
1073 BB->insert(I: BBBegin, MI: DbgMI);
1074 else {
1075 // Insert at the instruction, which may be in a different
1076 // block, if the block was split by a custom inserter.
1077 MachineBasicBlock::iterator Pos = MI;
1078 MI->getParent()->insert(I: Pos, MI: DbgMI);
1079 }
1080 }
1081 }
1082 if (DLI == DLE)
1083 break;
1084
1085 LastOrder = Order;
1086 }
1087 }
1088
1089 InsertPos = Emitter.getInsertPos();
1090 // In some cases, DBG_VALUEs might be inserted after the first terminator,
1091 // which results in an invalid MBB. If that happens, move the DBG_VALUEs
1092 // before the first terminator.
1093 MachineBasicBlock *InsertBB = Emitter.getBlock();
1094 auto FirstTerm = InsertBB->getFirstTerminator();
1095 if (FirstTerm != InsertBB->end()) {
1096 assert(!FirstTerm->isDebugValue() &&
1097 "first terminator cannot be a debug value");
1098 for (MachineInstr &MI : make_early_inc_range(
1099 Range: make_range(x: std::next(x: FirstTerm), y: InsertBB->end()))) {
1100 // Only scan up to insertion point.
1101 if (&MI == InsertPos)
1102 break;
1103
1104 if (!MI.isDebugValue())
1105 continue;
1106
1107 // The DBG_VALUE was referencing a value produced by a terminator. By
1108 // moving the DBG_VALUE, the referenced value also needs invalidating.
1109 MI.getOperand(i: 0).ChangeToRegister(Reg: 0, isDef: false);
1110 MI.moveBefore(MovePos: &*FirstTerm);
1111 }
1112 }
1113 return InsertBB;
1114}
1115
1116/// Return the basic block label.
1117std::string ScheduleDAGSDNodes::getDAGName() const {
1118 return "sunit-dag." + BB->getFullName();
1119}
1120