1//===- HexagonSubtarget.cpp - Hexagon Subtarget Information ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the Hexagon specific subclass of TargetSubtarget.
10//
11//===----------------------------------------------------------------------===//
12
13#include "HexagonSubtarget.h"
14#include "HexagonInstrInfo.h"
15#include "HexagonRegisterInfo.h"
16#include "MCTargetDesc/HexagonMCTargetDesc.h"
17#include "llvm/ADT/STLExtras.h"
18#include "llvm/ADT/SmallVector.h"
19#include "llvm/ADT/StringRef.h"
20#include "llvm/CodeGen/LibcallLoweringInfo.h"
21#include "llvm/CodeGen/MachineInstr.h"
22#include "llvm/CodeGen/MachineOperand.h"
23#include "llvm/CodeGen/MachineScheduler.h"
24#include "llvm/CodeGen/ScheduleDAG.h"
25#include "llvm/CodeGen/ScheduleDAGInstrs.h"
26#include "llvm/IR/IntrinsicsHexagon.h"
27#include "llvm/Support/CommandLine.h"
28#include "llvm/Support/ErrorHandling.h"
29#include "llvm/Target/TargetMachine.h"
30#include <algorithm>
31#include <cassert>
32#include <optional>
33
34using namespace llvm;
35
36#define DEBUG_TYPE "hexagon-subtarget"
37
38#define GET_SUBTARGETINFO_CTOR
39#define GET_SUBTARGETINFO_TARGET_DESC
40#include "HexagonGenSubtargetInfo.inc"
41
42static cl::opt<bool> EnableBSBSched("enable-bsb-sched", cl::Hidden,
43 cl::init(Val: true));
44
45static cl::opt<bool> EnableTCLatencySched("enable-tc-latency-sched", cl::Hidden,
46 cl::init(Val: false));
47
48static cl::opt<bool>
49 EnableDotCurSched("enable-cur-sched", cl::Hidden, cl::init(Val: true),
50 cl::desc("Enable the scheduler to generate .cur"));
51
52static cl::opt<bool>
53 DisableHexagonMISched("disable-hexagon-misched", cl::Hidden,
54 cl::desc("Disable Hexagon MI Scheduling"));
55
56static cl::opt<bool> OverrideLongCalls(
57 "hexagon-long-calls", cl::Hidden,
58 cl::desc("If present, forces/disables the use of long calls"));
59
60static cl::opt<bool>
61 EnablePredicatedCalls("hexagon-pred-calls", cl::Hidden,
62 cl::desc("Consider calls to be predicable"));
63
64static cl::opt<bool> SchedPredsCloser("sched-preds-closer", cl::Hidden,
65 cl::init(Val: true));
66
67static cl::opt<bool> SchedRetvalOptimization("sched-retval-optimization",
68 cl::Hidden, cl::init(Val: true));
69
70static cl::opt<bool> EnableCheckBankConflict(
71 "hexagon-check-bank-conflict", cl::Hidden, cl::init(Val: true),
72 cl::desc("Enable checking for cache bank conflicts"));
73
74HexagonSubtarget::HexagonSubtarget(const Triple &TT, StringRef CPU,
75 StringRef FS, const TargetMachine &TM)
76 : HexagonGenSubtargetInfo(TT, CPU, /*TuneCPU*/ CPU, FS),
77 OptLevel(TM.getOptLevel()),
78 CPUString(std::string(Hexagon_MC::selectHexagonCPU(CPU))),
79 TargetTriple(TT), InstrInfo(initializeSubtargetDependencies(CPU, FS)),
80 TLInfo(TM, *this), InstrItins(getInstrItineraryForCPU(CPU: CPUString)) {
81 Hexagon_MC::addArchSubtarget(STI: this, FS);
82 // Beware of the default constructor of InstrItineraryData: it will
83 // reset all members to 0.
84 assert(InstrItins.Itineraries != nullptr && "InstrItins not initialized");
85}
86
87HexagonSubtarget &
88HexagonSubtarget::initializeSubtargetDependencies(StringRef CPU, StringRef FS) {
89 std::optional<Hexagon::ArchEnum> ArchVer = Hexagon::getCpu(CPU: CPUString);
90 if (ArchVer)
91 HexagonArchVersion = *ArchVer;
92 else
93 llvm_unreachable("Unrecognized Hexagon processor version");
94
95 UseHVX128BOps = false;
96 UseHVX64BOps = false;
97 UseAudioOps = false;
98 UseLongCalls = false;
99
100 SubtargetFeatures Features(FS);
101
102 // Turn on QFloat if the HVX version is v68+.
103 // The function ParseSubtargetFeatures will set feature bits and initialize
104 // subtarget's variables all in one, so there isn't a good way to preprocess
105 // the feature string, other than by tinkering with it directly.
106 auto IsQFloatFS = [](StringRef F) {
107 return F == "+hvx-qfloat" || F == "-hvx-qfloat";
108 };
109 if (!llvm::count_if(Range: Features.getFeatures(), P: IsQFloatFS)) {
110 auto getHvxVersion = [&Features](StringRef FS) -> StringRef {
111 for (StringRef F : llvm::reverse(C: Features.getFeatures())) {
112 if (F.starts_with(Prefix: "+hvxv"))
113 return F;
114 }
115 for (StringRef F : llvm::reverse(C: Features.getFeatures())) {
116 if (F == "-hvx")
117 return StringRef();
118 if (F.starts_with(Prefix: "+hvx") || F == "-hvx")
119 return F.take_front(N: 4); // Return "+hvx" or "-hvx".
120 }
121 return StringRef();
122 };
123
124 bool AddQFloat = false;
125 StringRef HvxVer = getHvxVersion(FS);
126 if (HvxVer.starts_with(Prefix: "+hvxv")) {
127 int Ver = 0;
128 if (!HvxVer.drop_front(N: 5).consumeInteger(Radix: 10, Result&: Ver) && Ver >= 68)
129 AddQFloat = true;
130 } else if (HvxVer == "+hvx") {
131 if (hasV68Ops())
132 AddQFloat = true;
133 }
134
135 if (AddQFloat)
136 Features.AddFeature(String: "+hvx-qfloat");
137 }
138
139 std::string FeatureString = Features.getString();
140 ParseSubtargetFeatures(CPU: CPUString, /*TuneCPU*/ CPUString, FS: FeatureString);
141
142 // Resolve the shadow call stack pointer register. At most one "scs-reg-rN"
143 // feature may be given; R18 is the default. R18 is chosen because it is the
144 // lowest callee-saved register that neither the Hexagon Linux kernel (which
145 // reserves R19 for the thread-info pointer) nor code that reserves the upper
146 // callee-saved range already claims.
147 static_assert(Hexagon::R27 - Hexagon::R16 == 11,
148 "Callee-saved R16-R27 are assumed to be consecutive");
149 SCSPReg = Hexagon::R18;
150 bool SCSRegSelected = false;
151 for (unsigned Reg = Hexagon::R16; Reg <= Hexagon::R27; ++Reg) {
152 if (!SCSPointerRegister[Reg])
153 continue;
154 if (SCSRegSelected)
155 report_fatal_error(
156 reason: "Only one shadow call stack pointer register may be selected");
157 SCSPReg = Reg;
158 SCSRegSelected = true;
159 }
160
161 if (useHVXV68Ops())
162 UseHVXFloatingPoint = UseHVXIEEEFPOps || UseHVXQFloatOps;
163
164 if (UseHVXQFloatOps && UseHVXIEEEFPOps && UseHVXFloatingPoint)
165 LLVM_DEBUG(
166 dbgs() << "Behavior is undefined for simultaneous qfloat and ieee hvx codegen...");
167
168 if (OverrideLongCalls.getPosition())
169 UseLongCalls = OverrideLongCalls;
170
171 UseBSBScheduling = hasV60Ops() && EnableBSBSched;
172
173 if (isTinyCore()) {
174 // Tiny core has a single thread, so back-to-back scheduling is enabled by
175 // default.
176 if (!EnableBSBSched.getPosition())
177 UseBSBScheduling = false;
178 }
179
180 FeatureBitset FeatureBits = getFeatureBits();
181 if (HexagonDisableDuplex)
182 setFeatureBits(FeatureBits.reset(I: Hexagon::FeatureDuplex));
183 setFeatureBits(Hexagon_MC::completeHVXFeatures(FB: FeatureBits));
184
185 return *this;
186}
187
188void HexagonSubtarget::initLibcallLoweringInfo(
189 LibcallLoweringInfo &Info) const {
190 // The generic arithmetic/division helper routines (__adddf3, __divsi3, ...)
191 // exist in Hexagon's compiler-rt alongside the preferred __hexagon_*
192 // variants, so both are available. The __hexagon_* variant is the one that
193 // must be used; select it explicitly here.
194 static const struct {
195 const RTLIB::Libcall Op;
196 const RTLIB::LibcallImpl Impl;
197 } LibraryCalls[] = {
198 {.Op: RTLIB::SDIV_I32, .Impl: RTLIB::impl___hexagon_divsi3},
199 {.Op: RTLIB::SDIV_I64, .Impl: RTLIB::impl___hexagon_divdi3},
200 {.Op: RTLIB::UDIV_I32, .Impl: RTLIB::impl___hexagon_udivsi3},
201 {.Op: RTLIB::UDIV_I64, .Impl: RTLIB::impl___hexagon_udivdi3},
202 {.Op: RTLIB::SREM_I32, .Impl: RTLIB::impl___hexagon_modsi3},
203 {.Op: RTLIB::SREM_I64, .Impl: RTLIB::impl___hexagon_moddi3},
204 {.Op: RTLIB::UREM_I32, .Impl: RTLIB::impl___hexagon_umodsi3},
205 {.Op: RTLIB::UREM_I64, .Impl: RTLIB::impl___hexagon_umoddi3},
206 {.Op: RTLIB::ADD_F64, .Impl: RTLIB::impl___hexagon_adddf3},
207 {.Op: RTLIB::SUB_F64, .Impl: RTLIB::impl___hexagon_subdf3},
208 {.Op: RTLIB::MUL_F64, .Impl: RTLIB::impl___hexagon_muldf3},
209 {.Op: RTLIB::DIV_F64, .Impl: RTLIB::impl___hexagon_divdf3},
210 {.Op: RTLIB::DIV_F32, .Impl: RTLIB::impl___hexagon_divsf3},
211 };
212 for (const auto &LC : LibraryCalls)
213 Info.setLibcallImpl(Call: LC.Op, Impl: LC.Impl);
214}
215
216bool HexagonSubtarget::isHVXElementType(MVT Ty, bool IncludeBool) const {
217 if (!useHVXOps())
218 return false;
219 if (Ty.isVector())
220 Ty = Ty.getVectorElementType();
221 if (IncludeBool && Ty == MVT::i1)
222 return true;
223 ArrayRef<MVT> ElemTypes = getHVXElementTypes();
224 return llvm::is_contained(Range&: ElemTypes, Element: Ty);
225}
226
227bool HexagonSubtarget::isHVXVectorType(EVT VecTy, bool IncludeBool) const {
228 if (!VecTy.isSimple())
229 return false;
230 if (!VecTy.isVector() || !useHVXOps() || VecTy.isScalableVector())
231 return false;
232 MVT ElemTy = VecTy.getSimpleVT().getVectorElementType();
233 if (!IncludeBool && ElemTy == MVT::i1)
234 return false;
235
236 unsigned HwLen = getVectorLength();
237 unsigned NumElems = VecTy.getVectorNumElements();
238 ArrayRef<MVT> ElemTypes = getHVXElementTypes();
239
240 if (IncludeBool && ElemTy == MVT::i1) {
241 // Boolean HVX vector types are formed from regular HVX vector types
242 // by replacing the element type with i1.
243 for (MVT T : ElemTypes)
244 if (NumElems * T.getSizeInBits() == 8 * HwLen)
245 return true;
246 return false;
247 }
248
249 unsigned VecWidth = VecTy.getSizeInBits();
250 if (VecWidth != 8 * HwLen && VecWidth != 16 * HwLen)
251 return false;
252 return llvm::is_contained(Range&: ElemTypes, Element: ElemTy);
253}
254
255bool HexagonSubtarget::isTypeForHVX(Type *VecTy, bool IncludeBool) const {
256 if (!VecTy->isVectorTy() || isa<ScalableVectorType>(Val: VecTy))
257 return false;
258 // HLX types have their own legalization path and never go through HVX.
259 if (useHLXOps() && isTypeForHLX(VecTy))
260 return false;
261 // Avoid types like <2 x i32*>.
262 Type *ScalTy = VecTy->getScalarType();
263 if (!ScalTy->isIntegerTy() &&
264 !(ScalTy->isFloatingPointTy() && useHVXFloatingPoint()))
265 return false;
266 // The given type may be something like <17 x i32>, which is not MVT,
267 // but can be represented as (non-simple) EVT.
268 EVT Ty = EVT::getEVT(Ty: VecTy, /*HandleUnknown*/false);
269 if (!Ty.getVectorElementType().isSimple())
270 return false;
271
272 auto isHvxTy = [this, IncludeBool](MVT SimpleTy) {
273 if (isHVXVectorType(VecTy: SimpleTy, IncludeBool))
274 return true;
275 auto Action = getTargetLowering()->getPreferredVectorAction(VT: SimpleTy);
276 return Action == TargetLoweringBase::TypeWidenVector;
277 };
278
279 // Round up EVT to have power-of-2 elements, and keep checking if it
280 // qualifies for HVX, dividing it in half after each step.
281 MVT ElemTy = Ty.getVectorElementType().getSimpleVT();
282 unsigned VecLen = PowerOf2Ceil(A: Ty.getVectorNumElements());
283 while (VecLen > 1) {
284 MVT SimpleTy = MVT::getVectorVT(VT: ElemTy, NumElements: VecLen);
285 if (SimpleTy.isValid() && isHvxTy(SimpleTy))
286 return true;
287 VecLen /= 2;
288 }
289
290 return false;
291}
292
293bool HexagonSubtarget::isHLXElementType(MVT Ty) const {
294 if (!useHLXOps())
295 return false;
296 if (Ty.isVector())
297 Ty = Ty.getVectorElementType();
298 ArrayRef<MVT> ElemTypes = getHLXElementTypes();
299 return llvm::is_contained(Range&: ElemTypes, Element: Ty);
300}
301
302bool HexagonSubtarget::isHLXVectorType(EVT VecTy) const {
303 if (!VecTy.isSimple())
304 return false;
305 if (!VecTy.isVector() || !useHLXOps() || VecTy.isScalableVector())
306 return false;
307 MVT ElemTy = VecTy.getSimpleVT().getVectorElementType();
308
309 unsigned HwLen = getHLXVectorLength();
310 ArrayRef<MVT> ElemTypes = getHLXElementTypes();
311
312 unsigned VecWidth = VecTy.getSizeInBits();
313 if (VecWidth != 8 * HwLen && VecWidth != 16 * HwLen)
314 return false;
315 return llvm::is_contained(Range&: ElemTypes, Element: ElemTy);
316}
317
318bool HexagonSubtarget::isTypeForHLX(Type *VecTy) const {
319 if (!VecTy->isVectorTy() || isa<ScalableVectorType>(Val: VecTy))
320 return false;
321 // Avoid types like <2 x i32*>.
322 Type *ScalTy = VecTy->getScalarType();
323 if (!ScalTy->isIntegerTy() && !ScalTy->isFloatingPointTy())
324 return false;
325 // The given type may be something like <17 x i32>, which is not MVT,
326 // but can be represented as (non-simple) EVT.
327 EVT Ty = EVT::getEVT(Ty: VecTy, /*HandleUnknown*/ false);
328 if (!Ty.getVectorElementType().isSimple())
329 return false;
330
331 auto isHLXTy = [this](MVT SimpleTy) {
332 if (isHLXVectorType(VecTy: SimpleTy))
333 return true;
334 // No widening for now
335 return false;
336 };
337
338 // Round up EVT to have power-of-2 elements, and keep checking if it
339 // qualifies for HLX, dividing it in half after each step.
340 MVT ElemTy = Ty.getVectorElementType().getSimpleVT();
341 unsigned VecLen = PowerOf2Ceil(A: Ty.getVectorNumElements());
342 while (VecLen > 1) {
343 MVT SimpleTy = MVT::getVectorVT(VT: ElemTy, NumElements: VecLen);
344 if (SimpleTy.isValid() && isHLXTy(SimpleTy))
345 return true;
346 VecLen /= 2;
347 }
348
349 return false;
350}
351
352void HexagonSubtarget::UsrOverflowMutation::apply(ScheduleDAGInstrs *DAG) {
353 for (SUnit &SU : DAG->SUnits) {
354 if (!SU.isInstr())
355 continue;
356 SmallVector<SDep, 4> Erase;
357 for (auto &D : SU.Preds)
358 if (D.getKind() == SDep::Output && D.getReg() == Hexagon::USR_OVF)
359 Erase.push_back(Elt: D);
360 for (auto &E : Erase)
361 SU.removePred(D: E);
362 }
363}
364
365void HexagonSubtarget::HVXMemLatencyMutation::apply(ScheduleDAGInstrs *DAG) {
366 for (SUnit &SU : DAG->SUnits) {
367 // Update the latency of chain edges between v60 vector load or store
368 // instructions to be 1. These instruction cannot be scheduled in the
369 // same packet.
370 MachineInstr &MI1 = *SU.getInstr();
371 auto *QII = static_cast<const HexagonInstrInfo*>(DAG->TII);
372 bool IsStoreMI1 = MI1.mayStore();
373 bool IsLoadMI1 = MI1.mayLoad();
374 if (!QII->isHVXVec(MI: MI1) || !(IsStoreMI1 || IsLoadMI1))
375 continue;
376 for (SDep &SI : SU.Succs) {
377 if (SI.getKind() != SDep::Order || SI.getLatency() != 0)
378 continue;
379 MachineInstr &MI2 = *SI.getSUnit()->getInstr();
380 if (!QII->isHVXVec(MI: MI2))
381 continue;
382 if ((IsStoreMI1 && MI2.mayStore()) || (IsLoadMI1 && MI2.mayLoad())) {
383 SI.setLatency(1);
384 SU.setHeightDirty();
385 // Change the dependence in the opposite direction too.
386 for (SDep &PI : SI.getSUnit()->Preds) {
387 if (PI.getSUnit() != &SU || PI.getKind() != SDep::Order)
388 continue;
389 PI.setLatency(1);
390 SI.getSUnit()->setDepthDirty();
391 }
392 }
393 }
394 }
395}
396
397// Ensure that HMX activate and weight instructions are scheduled together.
398// The main idea is to:
399// 1. Reduce any latency between corresponding A/W to 0.
400// 2. Make the depths of corresponding A/W instructions be the same. (The
401// post-RA scheduler is a top-down list scheduler.)
402void HexagonSubtarget::HMXActivateWeightMutation::apply(
403 ScheduleDAGInstrs *DAG) {
404 if (DAG->SUnits.empty())
405 return;
406
407 auto &HII = static_cast<const HexagonInstrInfo &>(*DAG->TII);
408 DenseMap<SUnit *, SUnit *> AWMap; // Map: Activation <-> Weight.
409 DenseSet<SUnit *> First; // Which of the two came first.
410 // The SUnits in the DAG follow the order of instructions. The code below
411 // depends on it to pair up activation and weight instructions.
412 SUnit *A = nullptr, *W = nullptr;
413 for (SUnit &SU : DAG->SUnits) {
414 MachineInstr *MI = SU.getInstr();
415 if (HII.isHMXActivateInstr(MI: *MI)) {
416 assert(A == nullptr && "Unexpected activation instruction");
417 A = &SU;
418 if (!W)
419 First.insert(V: A);
420 } else if (HII.isHMXWeightInstr(MI: *MI)) {
421 assert(W == nullptr && "Unexpected weight instruction");
422 W = &SU;
423 if (!A)
424 First.insert(V: W);
425 }
426 if (A && W) {
427 AWMap.insert(KV: {A, W});
428 AWMap.insert(KV: {W, A});
429 A = W = nullptr;
430 }
431 }
432 assert(A == nullptr && "Unmatched activation instruction");
433 assert(W == nullptr && "Unmatched weight instruction");
434
435 // Reset the latency between matching instructions to 0.
436 for (SUnit *FirstSU : First) {
437 SUnit *SecondSU = AWMap.lookup(Val: FirstSU);
438 // Set the successor latency to 0.
439 for (SDep &D : FirstSU->Succs) {
440 if (D.getSUnit() == SecondSU) {
441 D.setLatency(0);
442 FirstSU->setHeightDirty();
443 }
444 }
445 // Set the predecessor latency to 0.
446 for (SDep &D : SecondSU->Preds) {
447 if (D.getSUnit() == FirstSU) {
448 D.setLatency(0);
449 SecondSU->setDepthDirty();
450 }
451 }
452 }
453
454 SUnit *Entry = &DAG->SUnits[0];
455 assert(Entry->getDepth() == 0);
456
457 // For each pair, reset the depth (max distance from the entry) to be the
458 // max of the two. We can't just set the depth directly, because the post-RA
459 // scheduler recalculates it during scheduling. There are also assertions
460 // in it that can fire when the depth on the node is inconsistent with the
461 // calculations.
462 // To avoid this issue, make the depth "real" by adding a fake edge from
463 // the Entry node (which is assumed to have a depth of 0).
464 // Adding a new edge will mark the depths of successors (recursively)
465 // as dirty, potentially including subsequent pairs. The "getDepth" call
466 // will recompute a dirty depth, so it's important to iterate over the nodes
467 // in such a way that predecessors are visited first.
468 //
469 // Skip the first SUnit in case it's an A/W instruction, so that we don't
470 // try to create an edge from that node to itself.
471 for (SUnit &SU : MutableArrayRef<SUnit>(DAG->SUnits).drop_front()) {
472 if (!First.count(V: &SU))
473 continue;
474 SUnit *FirstSU = &SU;
475 SUnit *SecondSU = AWMap.lookup(Val: FirstSU);
476 unsigned MaxDepth = std::max(a: FirstSU->getDepth(), b: SecondSU->getDepth());
477 SDep DepthEdge(Entry, SDep::Artificial);
478 DepthEdge.setLatency(MaxDepth);
479 FirstSU->addPred(D: DepthEdge);
480 SecondSU->addPred(D: DepthEdge);
481 }
482}
483
484// Check if a call and subsequent A2_tfrpi instructions should maintain
485// scheduling affinity. We are looking for the TFRI to be consumed in
486// the next instruction. This should help reduce the instances of
487// double register pairs being allocated and scheduled before a call
488// when not used until after the call. This situation is exacerbated
489// by the fact that we allocate the pair from the callee saves list,
490// leading to excess spills and restores.
491bool HexagonSubtarget::CallMutation::shouldTFRICallBind(
492 const HexagonInstrInfo &HII, const SUnit &Inst1,
493 const SUnit &Inst2) const {
494 if (Inst1.getInstr()->getOpcode() != Hexagon::A2_tfrpi)
495 return false;
496
497 // TypeXTYPE are 64 bit operations.
498 unsigned Type = HII.getType(MI: *Inst2.getInstr());
499 return Type == HexagonII::TypeS_2op || Type == HexagonII::TypeS_3op ||
500 Type == HexagonII::TypeALU64 || Type == HexagonII::TypeM;
501}
502
503void HexagonSubtarget::CallMutation::apply(ScheduleDAGInstrs *DAGInstrs) {
504 ScheduleDAGMI *DAG = static_cast<ScheduleDAGMI*>(DAGInstrs);
505 SUnit* LastSequentialCall = nullptr;
506 // Map from virtual register to physical register from the copy.
507 DenseMap<unsigned, unsigned> VRegHoldingReg;
508 // Map from the physical register to the instruction that uses virtual
509 // register. This is used to create the barrier edge.
510 DenseMap<unsigned, SUnit *> LastVRegUse;
511 auto &TRI = *DAG->MF.getSubtarget().getRegisterInfo();
512 auto &HII = *DAG->MF.getSubtarget<HexagonSubtarget>().getInstrInfo();
513
514 // Currently we only catch the situation when compare gets scheduled
515 // before preceding call.
516 for (unsigned su = 0, e = DAG->SUnits.size(); su != e; ++su) {
517 // Remember the call.
518 if (DAG->SUnits[su].getInstr()->isCall())
519 LastSequentialCall = &DAG->SUnits[su];
520 // Look for a compare that defines a predicate.
521 else if (DAG->SUnits[su].getInstr()->isCompare() && LastSequentialCall)
522 DAG->addEdge(SuccSU: &DAG->SUnits[su], PredDep: SDep(LastSequentialCall, SDep::Barrier));
523 // Look for call and tfri* instructions.
524 else if (SchedPredsCloser && LastSequentialCall && su > 1 && su < e-1 &&
525 shouldTFRICallBind(HII, Inst1: DAG->SUnits[su], Inst2: DAG->SUnits[su+1]))
526 DAG->addEdge(SuccSU: &DAG->SUnits[su], PredDep: SDep(&DAG->SUnits[su-1], SDep::Barrier));
527 // Prevent redundant register copies due to reads and writes of physical
528 // registers. The original motivation for this was the code generated
529 // between two calls, which are caused both the return value and the
530 // argument for the next call being in %r0.
531 // Example:
532 // 1: <call1>
533 // 2: %vreg = COPY %r0
534 // 3: <use of %vreg>
535 // 4: %r0 = ...
536 // 5: <call2>
537 // The scheduler would often swap 3 and 4, so an additional register is
538 // needed. This code inserts a Barrier dependence between 3 & 4 to prevent
539 // this.
540 // The code below checks for all the physical registers, not just R0/D0/V0.
541 else if (SchedRetvalOptimization) {
542 const MachineInstr *MI = DAG->SUnits[su].getInstr();
543 if (MI->isCopy() && MI->getOperand(i: 1).getReg().isPhysical()) {
544 // %vregX = COPY %r0
545 VRegHoldingReg[MI->getOperand(i: 0).getReg()] = MI->getOperand(i: 1).getReg();
546 LastVRegUse.erase(Val: MI->getOperand(i: 1).getReg());
547 } else {
548 for (const MachineOperand &MO : MI->operands()) {
549 if (!MO.isReg())
550 continue;
551 if (MO.isUse() && !MI->isCopy() &&
552 VRegHoldingReg.count(Val: MO.getReg())) {
553 // <use of %vregX>
554 LastVRegUse[VRegHoldingReg[MO.getReg()]] = &DAG->SUnits[su];
555 } else if (MO.isDef() && MO.getReg().isPhysical()) {
556 for (MCRegAliasIterator AI(MO.getReg(), &TRI, true); AI.isValid();
557 ++AI) {
558 if (auto It = LastVRegUse.find(Val: *AI); It != LastVRegUse.end()) {
559 if (It->second != &DAG->SUnits[su])
560 // %r0 = ...
561 DAG->addEdge(SuccSU: &DAG->SUnits[su],
562 PredDep: SDep(It->second, SDep::Barrier));
563 LastVRegUse.erase(I: It);
564 }
565 }
566 }
567 }
568 }
569 }
570 }
571}
572
573void HexagonSubtarget::BankConflictMutation::apply(ScheduleDAGInstrs *DAG) {
574 if (!EnableCheckBankConflict)
575 return;
576
577 const auto &HII = static_cast<const HexagonInstrInfo&>(*DAG->TII);
578
579 // Create artificial edges between loads that could likely cause a bank
580 // conflict. Since such loads would normally not have any dependency
581 // between them, we cannot rely on existing edges.
582 for (unsigned i = 0, e = DAG->SUnits.size(); i != e; ++i) {
583 SUnit &S0 = DAG->SUnits[i];
584 MachineInstr &L0 = *S0.getInstr();
585 if (!L0.mayLoad() || L0.mayStore() ||
586 HII.getAddrMode(MI: L0) != HexagonII::BaseImmOffset)
587 continue;
588 int64_t Offset0;
589 LocationSize Size0 = LocationSize::precise(Value: 0);
590 MachineOperand *BaseOp0 = HII.getBaseAndOffset(MI: L0, Offset&: Offset0, AccessSize&: Size0);
591 // Is the access size is longer than the L1 cache line, skip the check.
592 if (BaseOp0 == nullptr || !BaseOp0->isReg() || !Size0.hasValue() ||
593 Size0.getValue() >= 32)
594 continue;
595 // Scan only up to 32 instructions ahead (to avoid n^2 complexity).
596 for (unsigned j = i+1, m = std::min(a: i+32, b: e); j != m; ++j) {
597 SUnit &S1 = DAG->SUnits[j];
598 MachineInstr &L1 = *S1.getInstr();
599 if (!L1.mayLoad() || L1.mayStore() ||
600 HII.getAddrMode(MI: L1) != HexagonII::BaseImmOffset)
601 continue;
602 int64_t Offset1;
603 LocationSize Size1 = LocationSize::precise(Value: 0);
604 MachineOperand *BaseOp1 = HII.getBaseAndOffset(MI: L1, Offset&: Offset1, AccessSize&: Size1);
605 if (BaseOp1 == nullptr || !BaseOp1->isReg() || !Size0.hasValue() ||
606 Size1.getValue() >= 32 || BaseOp0->getReg() != BaseOp1->getReg())
607 continue;
608 // Check bits 3 and 4 of the offset: if they differ, a bank conflict
609 // is unlikely.
610 if (((Offset0 ^ Offset1) & 0x18) != 0)
611 continue;
612 // Bits 3 and 4 are the same, add an artificial edge and set extra
613 // latency.
614 SDep A(&S0, SDep::Artificial);
615 A.setLatency(1);
616 S1.addPred(D: A, Required: true);
617 }
618 }
619}
620
621void HexagonSubtarget::HLXPseudoMutation::apply(ScheduleDAGInstrs *DAG) {
622 const auto &HII = static_cast<const HexagonInstrInfo &>(*DAG->TII);
623 const auto &HST = DAG->MF.getSubtarget<HexagonSubtarget>();
624 if (!HST.useHLXOps())
625 return;
626
627 auto setLatency = [](SUnit &From, SUnit &To) {
628 for (SDep &Dep : From.Succs)
629 if (Dep.getSUnit() == &To && Dep.getKind() == SDep::Data)
630 Dep.setLatency(0);
631 for (SDep &Dep : To.Preds)
632 if (Dep.getSUnit() == &From && Dep.getKind() == SDep::Data)
633 Dep.setLatency(0);
634 };
635
636 for (SUnit &SU : DAG->SUnits) {
637 if (!SU.isInstr())
638 continue;
639
640 MachineInstr &MI = *SU.getInstr();
641 if (HII.isTmpLoadPseudo(MI)) {
642 for (SDep &Dep : SU.Succs)
643 if (Dep.getKind() == SDep::Data)
644 setLatency(SU, *Dep.getSUnit());
645 } else if (HII.isNewStorePseudo(MI)) {
646 for (SDep &Dep : SU.Preds)
647 if (Dep.getKind() == SDep::Data)
648 setLatency(*Dep.getSUnit(), SU);
649 }
650 }
651}
652
653/// Enable use of alias analysis during code generation (during MI
654/// scheduling, DAGCombine, etc.).
655bool HexagonSubtarget::useAA() const {
656 if (OptLevel != CodeGenOptLevel::None)
657 return true;
658 return false;
659}
660
661/// Perform target specific adjustments to the latency of a schedule
662/// dependency.
663void HexagonSubtarget::adjustSchedDependency(
664 SUnit *Src, int SrcOpIdx, SUnit *Dst, int DstOpIdx, SDep &Dep,
665 const TargetSchedModel *SchedModel) const {
666 if (!Src->isInstr() || !Dst->isInstr())
667 return;
668
669 MachineInstr *SrcInst = Src->getInstr();
670 MachineInstr *DstInst = Dst->getInstr();
671 const HexagonInstrInfo *QII = getInstrInfo();
672
673 // Instructions with .new operands have zero latency.
674 SmallPtrSet<SUnit *, 4> ExclSrc;
675 SmallPtrSet<SUnit *, 4> ExclDst;
676 if (QII->canExecuteInBundle(First: *SrcInst, Second: *DstInst) &&
677 isBestZeroLatency(Src, Dst, TII: QII, ExclSrc, ExclDst)) {
678 Dep.setLatency(0);
679 return;
680 }
681
682 // Set the latency for a copy to zero since we hope that is will get
683 // removed.
684 if (DstInst->isCopy())
685 Dep.setLatency(0);
686
687 // If it's a REG_SEQUENCE/COPY, use its destination instruction to determine
688 // the correct latency.
689 // If there are multiple uses of the def of COPY/REG_SEQUENCE, set the latency
690 // only if the latencies on all the uses are equal, otherwise set it to
691 // default.
692 if ((DstInst->isRegSequence() || DstInst->isCopy())) {
693 Register DReg = DstInst->getOperand(i: 0).getReg();
694 std::optional<unsigned> DLatency;
695 for (const auto &DDep : Dst->Succs) {
696 MachineInstr *DDst = DDep.getSUnit()->getInstr();
697 int UseIdx = -1;
698 for (unsigned OpNum = 0; OpNum < DDst->getNumOperands(); OpNum++) {
699 const MachineOperand &MO = DDst->getOperand(i: OpNum);
700 if (MO.isReg() && MO.getReg() && MO.isUse() && MO.getReg() == DReg) {
701 UseIdx = OpNum;
702 break;
703 }
704 }
705
706 if (UseIdx == -1)
707 continue;
708
709 std::optional<unsigned> Latency =
710 InstrInfo.getOperandLatency(ItinData: &InstrItins, DefMI: *SrcInst, DefIdx: 0, UseMI: *DDst, UseIdx);
711
712 // Set DLatency for the first time.
713 if (!DLatency)
714 DLatency = Latency;
715
716 // For multiple uses, if the Latency is different across uses, reset
717 // DLatency.
718 if (DLatency != Latency) {
719 DLatency = std::nullopt;
720 break;
721 }
722 }
723 Dep.setLatency(DLatency.value_or(u: 0));
724 }
725
726 // Try to schedule uses near definitions to generate .cur.
727 ExclSrc.clear();
728 ExclDst.clear();
729 if (EnableDotCurSched && QII->isToBeScheduledASAP(MI1: *SrcInst, MI2: *DstInst) &&
730 isBestZeroLatency(Src, Dst, TII: QII, ExclSrc, ExclDst)) {
731 Dep.setLatency(0);
732 return;
733 }
734 int Latency = Dep.getLatency();
735 bool IsArtificial = Dep.isArtificial();
736 Latency = updateLatency(SrcInst&: *SrcInst, DstInst&: *DstInst, IsArtificial, Latency);
737 Dep.setLatency(Latency);
738}
739
740void HexagonSubtarget::getPostRAMutations(
741 std::vector<std::unique_ptr<ScheduleDAGMutation>> &Mutations) const {
742 Mutations.push_back(x: std::make_unique<UsrOverflowMutation>());
743 Mutations.push_back(x: std::make_unique<HVXMemLatencyMutation>());
744 Mutations.push_back(x: std::make_unique<HMXActivateWeightMutation>());
745 Mutations.push_back(x: std::make_unique<BankConflictMutation>());
746 Mutations.push_back(x: std::make_unique<HLXPseudoMutation>());
747}
748
749void HexagonSubtarget::getSMSMutations(
750 std::vector<std::unique_ptr<ScheduleDAGMutation>> &Mutations) const {
751 Mutations.push_back(x: std::make_unique<UsrOverflowMutation>());
752 Mutations.push_back(x: std::make_unique<HVXMemLatencyMutation>());
753 Mutations.push_back(x: std::make_unique<HMXActivateWeightMutation>());
754}
755
756// Pin the vtable to this file.
757void HexagonSubtarget::anchor() {}
758
759bool HexagonSubtarget::enableMachineScheduler() const {
760 if (DisableHexagonMISched.getNumOccurrences())
761 return !DisableHexagonMISched;
762 return true;
763}
764
765bool HexagonSubtarget::usePredicatedCalls() const {
766 return EnablePredicatedCalls;
767}
768
769int HexagonSubtarget::updateLatency(MachineInstr &SrcInst,
770 MachineInstr &DstInst, bool IsArtificial,
771 int Latency) const {
772 if (IsArtificial)
773 return 1;
774 if (!hasV60Ops())
775 return Latency;
776
777 const HexagonInstrInfo &QII = *getInstrInfo();
778 // BSB scheduling.
779 if (QII.isHVXVec(MI: SrcInst) || useBSBScheduling())
780 Latency = (Latency + 1) >> 1;
781 return Latency;
782}
783
784void HexagonSubtarget::restoreLatency(SUnit *Src, SUnit *Dst) const {
785 MachineInstr *SrcI = Src->getInstr();
786 for (auto &I : Src->Succs) {
787 if (!I.isAssignedRegDep() || I.getSUnit() != Dst)
788 continue;
789 Register DepR = I.getReg();
790 int DefIdx = -1;
791 for (unsigned OpNum = 0; OpNum < SrcI->getNumOperands(); OpNum++) {
792 const MachineOperand &MO = SrcI->getOperand(i: OpNum);
793 bool IsSameOrSubReg = false;
794 if (MO.isReg()) {
795 Register MOReg = MO.getReg();
796 if (DepR.isVirtual()) {
797 IsSameOrSubReg = (MOReg == DepR);
798 } else {
799 IsSameOrSubReg = getRegisterInfo()->isSubRegisterEq(RegA: DepR, RegB: MOReg);
800 }
801 if (MO.isDef() && IsSameOrSubReg)
802 DefIdx = OpNum;
803 }
804 }
805 assert(DefIdx >= 0 && "Def Reg not found in Src MI");
806 MachineInstr *DstI = Dst->getInstr();
807 SDep T = I;
808 for (unsigned OpNum = 0; OpNum < DstI->getNumOperands(); OpNum++) {
809 const MachineOperand &MO = DstI->getOperand(i: OpNum);
810 if (MO.isReg() && MO.isUse() && MO.getReg() == DepR) {
811 std::optional<unsigned> Latency = InstrInfo.getOperandLatency(
812 ItinData: &InstrItins, DefMI: *SrcI, DefIdx, UseMI: *DstI, UseIdx: OpNum);
813
814 // For some instructions (ex: COPY), we might end up with < 0 latency
815 // as they don't have any Itinerary class associated with them.
816 if (!Latency)
817 Latency = 0;
818 bool IsArtificial = I.isArtificial();
819 Latency = updateLatency(SrcInst&: *SrcI, DstInst&: *DstI, IsArtificial, Latency: *Latency);
820 I.setLatency(*Latency);
821 }
822 }
823
824 // Update the latency of opposite edge too.
825 T.setSUnit(Src);
826 auto F = find(Range&: Dst->Preds, Val: T);
827 assert(F != Dst->Preds.end());
828 F->setLatency(I.getLatency());
829 }
830}
831
832/// Change the latency between the two SUnits.
833void HexagonSubtarget::changeLatency(SUnit *Src, SUnit *Dst, unsigned Lat)
834 const {
835 for (auto &I : Src->Succs) {
836 if (!I.isAssignedRegDep() || I.getSUnit() != Dst)
837 continue;
838 SDep T = I;
839 I.setLatency(Lat);
840
841 // Update the latency of opposite edge too.
842 T.setSUnit(Src);
843 auto F = find(Range&: Dst->Preds, Val: T);
844 assert(F != Dst->Preds.end());
845 F->setLatency(Lat);
846 }
847}
848
849/// If the SUnit has a zero latency edge, return the other SUnit.
850static SUnit *getZeroLatency(SUnit *N, SmallVector<SDep, 4> &Deps) {
851 for (auto &I : Deps)
852 if (I.isAssignedRegDep() && I.getLatency() == 0 &&
853 !I.getSUnit()->getInstr()->isPseudo())
854 return I.getSUnit();
855 return nullptr;
856}
857
858// Return true if these are the best two instructions to schedule
859// together with a zero latency. Only one dependence should have a zero
860// latency. If there are multiple choices, choose the best, and change
861// the others, if needed.
862bool HexagonSubtarget::isBestZeroLatency(
863 SUnit *Src, SUnit *Dst, const HexagonInstrInfo *TII,
864 SmallPtrSet<SUnit *, 4> &ExclSrc, SmallPtrSet<SUnit *, 4> &ExclDst) const {
865 MachineInstr &SrcInst = *Src->getInstr();
866 MachineInstr &DstInst = *Dst->getInstr();
867
868 // Ignore Boundary SU nodes as these have null instructions.
869 if (Dst->isBoundaryNode())
870 return false;
871
872 if (SrcInst.isPHI() || DstInst.isPHI())
873 return false;
874
875 if (!TII->isToBeScheduledASAP(MI1: SrcInst, MI2: DstInst) &&
876 !TII->canExecuteInBundle(First: SrcInst, Second: DstInst))
877 return false;
878
879 // The architecture doesn't allow three dependent instructions in the same
880 // packet. So, if the destination has a zero latency successor, then it's
881 // not a candidate for a zero latency predecessor.
882 if (getZeroLatency(N: Dst, Deps&: Dst->Succs) != nullptr)
883 return false;
884
885 // Check if the Dst instruction is the best candidate first.
886 SUnit *Best = nullptr;
887 SUnit *DstBest = nullptr;
888 SUnit *SrcBest = getZeroLatency(N: Dst, Deps&: Dst->Preds);
889 if (SrcBest == nullptr || Src->NodeNum >= SrcBest->NodeNum) {
890 // Check that Src doesn't have a better candidate.
891 DstBest = getZeroLatency(N: Src, Deps&: Src->Succs);
892 if (DstBest == nullptr || Dst->NodeNum <= DstBest->NodeNum)
893 Best = Dst;
894 }
895 if (Best != Dst)
896 return false;
897
898 // The caller frequently adds the same dependence twice. If so, then
899 // return true for this case too.
900 if ((Src == SrcBest && Dst == DstBest ) ||
901 (SrcBest == nullptr && Dst == DstBest) ||
902 (Src == SrcBest && Dst == nullptr))
903 return true;
904
905 // Reassign the latency for the previous bests, which requires setting
906 // the dependence edge in both directions.
907 if (SrcBest != nullptr) {
908 if (!hasV60Ops())
909 changeLatency(Src: SrcBest, Dst, Lat: 1);
910 else
911 restoreLatency(Src: SrcBest, Dst);
912 }
913 if (DstBest != nullptr) {
914 if (!hasV60Ops())
915 changeLatency(Src, Dst: DstBest, Lat: 1);
916 else
917 restoreLatency(Src, Dst: DstBest);
918 }
919
920 // Attempt to find another opportunity for zero latency in a different
921 // dependence.
922 if (SrcBest && DstBest)
923 // If there is an edge from SrcBest to DstBst, then try to change that
924 // to 0 now.
925 changeLatency(Src: SrcBest, Dst: DstBest, Lat: 0);
926 else if (DstBest) {
927 // Check if the previous best destination instruction has a new zero
928 // latency dependence opportunity.
929 ExclSrc.insert(Ptr: Src);
930 for (auto &I : DstBest->Preds)
931 if (ExclSrc.count(Ptr: I.getSUnit()) == 0 &&
932 isBestZeroLatency(Src: I.getSUnit(), Dst: DstBest, TII, ExclSrc, ExclDst))
933 changeLatency(Src: I.getSUnit(), Dst: DstBest, Lat: 0);
934 } else if (SrcBest) {
935 // Check if previous best source instruction has a new zero latency
936 // dependence opportunity.
937 ExclDst.insert(Ptr: Dst);
938 for (auto &I : SrcBest->Succs)
939 if (ExclDst.count(Ptr: I.getSUnit()) == 0 &&
940 isBestZeroLatency(Src: SrcBest, Dst: I.getSUnit(), TII, ExclSrc, ExclDst))
941 changeLatency(Src: SrcBest, Dst: I.getSUnit(), Lat: 0);
942 }
943
944 return true;
945}
946
947unsigned HexagonSubtarget::getL1CacheLineSize() const {
948 return 32;
949}
950
951unsigned HexagonSubtarget::getL1PrefetchDistance() const {
952 return 32;
953}
954
955bool HexagonSubtarget::enableSubRegLiveness() const { return true; }
956
957Intrinsic::ID HexagonSubtarget::getIntrinsicId(unsigned Opc) const {
958 struct Scalar {
959 unsigned Opcode;
960 Intrinsic::ID IntId;
961 };
962 struct Hvx {
963 unsigned Opcode;
964 Intrinsic::ID Int64Id, Int128Id;
965 };
966
967 static Scalar ScalarInts[] = {
968#define GET_SCALAR_INTRINSICS
969#include "HexagonDepInstrIntrinsics.inc"
970#undef GET_SCALAR_INTRINSICS
971 };
972
973 static Hvx HvxInts[] = {
974#define GET_HVX_INTRINSICS
975#include "HexagonDepInstrIntrinsics.inc"
976#undef GET_HVX_INTRINSICS
977 };
978
979 const auto CmpOpcode = [](auto A, auto B) { return A.Opcode < B.Opcode; };
980 [[maybe_unused]] static bool SortedScalar =
981 (llvm::sort(C&: ScalarInts, Comp: CmpOpcode), true);
982 [[maybe_unused]] static bool SortedHvx =
983 (llvm::sort(C&: HvxInts, Comp: CmpOpcode), true);
984
985 auto [BS, ES] = std::make_pair(x: std::begin(arr&: ScalarInts), y: std::end(arr&: ScalarInts));
986 auto [BH, EH] = std::make_pair(x: std::begin(arr&: HvxInts), y: std::end(arr&: HvxInts));
987
988 auto FoundScalar = std::lower_bound(first: BS, last: ES, val: Scalar{.Opcode: Opc, .IntId: 0}, comp: CmpOpcode);
989 if (FoundScalar != ES && FoundScalar->Opcode == Opc)
990 return FoundScalar->IntId;
991
992 auto FoundHvx = std::lower_bound(first: BH, last: EH, val: Hvx{.Opcode: Opc, .Int64Id: 0, .Int128Id: 0}, comp: CmpOpcode);
993 if (FoundHvx != EH && FoundHvx->Opcode == Opc) {
994 unsigned HwLen = getVectorLength();
995 if (HwLen == 64)
996 return FoundHvx->Int64Id;
997 if (HwLen == 128)
998 return FoundHvx->Int128Id;
999 }
1000
1001 std::string error = "Invalid opcode (" + std::to_string(val: Opc) + ")";
1002 llvm_unreachable(error.c_str());
1003 return 0;
1004}
1005