1//===--- AMDGPUBarrierLatency.cpp - AMDGPU Barrier Latency ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This file contains a DAG scheduling mutation to add latency to:
10/// 1. Barrier edges between ATOMIC_FENCE instructions and preceding
11/// memory accesses potentially affected by the fence.
12/// This encourages the scheduling of more instructions before
13/// ATOMIC_FENCE instructions. ATOMIC_FENCE instructions may
14/// introduce wait counting or indicate an impending S_BARRIER
15/// wait. Having more instructions in-flight across these
16/// constructs improves latency hiding.
17/// 2. Barrier edges from S_BARRIER_SIGNAL to S_BARRIER_WAIT.
18/// This encourages independent work to be scheduled between
19/// signal and wait, hiding barrier synchronization latency.
20//
21//===----------------------------------------------------------------------===//
22
23#include "AMDGPUBarrierLatency.h"
24#include "GCNSubtarget.h"
25#include "SIInstrInfo.h"
26#include "llvm/CodeGen/ScheduleDAGInstrs.h"
27#include "llvm/TargetParser/AtomicScope.h"
28
29using namespace llvm;
30
31namespace {
32
33class BarrierLatency : public ScheduleDAGMutation {
34private:
35 SmallSet<SyncScope::ID, 4> IgnoredScopes;
36
37public:
38 BarrierLatency(MachineFunction *MF) {
39 LLVMContext &Context = MF->getFunction().getContext();
40 const Triple &TT = MF->getSubtarget<GCNSubtarget>().getTargetTriple();
41 auto ScopeID = [&](AtomicScope Scope, bool OneAS) {
42 return Context.getOrInsertSyncScopeID(
43 SSN: *getAtomicScopeIRString(T: TT, S: Scope, IsSingleAddressSpace: OneAS));
44 };
45 IgnoredScopes.insert(V: SyncScope::SingleThread);
46 IgnoredScopes.insert(V: ScopeID(AtomicScope::Wavefront, /*OneAS=*/false));
47 IgnoredScopes.insert(V: ScopeID(AtomicScope::Wavefront, /*OneAS=*/true));
48 IgnoredScopes.insert(V: ScopeID(AtomicScope::Single, /*OneAS=*/true));
49
50 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
51 bool TgSplit =
52 ST.hasTgSplitSupport() && AMDGPU::isTgSplitEnabled(F: MF->getFunction());
53 if (!ST.requiresWaitOnWorkgroupReleaseFence(TgSplit)) {
54 // Prior to GFX10 workgroup scope does not normally require waitcnts
55 IgnoredScopes.insert(V: ScopeID(AtomicScope::Workgroup, /*OneAS=*/false));
56 }
57 }
58 void apply(ScheduleDAGInstrs *DAG) override;
59};
60
61void addLatencyToEdge(SDep &PredDep, SUnit &SU, unsigned Latency) {
62 SUnit *PredSU = PredDep.getSUnit();
63 SDep ForwardD = PredDep;
64 ForwardD.setSUnit(&SU);
65 for (SDep &SuccDep : PredSU->Succs) {
66 if (SuccDep == ForwardD) {
67 SuccDep.setLatency(SuccDep.getLatency() + Latency);
68 break;
69 }
70 }
71 PredDep.setLatency(PredDep.getLatency() + Latency);
72 PredSU->setDepthDirty();
73 SU.setDepthDirty();
74}
75
76void setLatencyForEdge(SDep &PredDep, SUnit &SU, unsigned Latency) {
77 SUnit *PredSU = PredDep.getSUnit();
78 SDep ForwardD = PredDep;
79 ForwardD.setSUnit(&SU);
80 for (SDep &SuccDep : PredSU->Succs) {
81 if (SuccDep == ForwardD) {
82 SuccDep.setLatency(Latency);
83 break;
84 }
85 }
86 PredDep.setLatency(Latency);
87 PredSU->setDepthDirty();
88 SU.setDepthDirty();
89}
90
91void BarrierLatency::apply(ScheduleDAGInstrs *DAG) {
92 const SIInstrInfo *TII = static_cast<const SIInstrInfo *>(DAG->TII);
93 constexpr unsigned FenceLatency = 2000;
94 const unsigned BarrierSignalWaitLatency = DAG->MF.getSubtarget<GCNSubtarget>()
95 .getCLOpts()
96 .barrier_signal_wait_latency;
97 SmallVector<SUnit *, 8> RegionTDM;
98 SmallVector<SUnit *, 8> RegionAsync;
99 const TargetSchedModel *SchedModel = DAG->getSchedModel();
100
101 for (SUnit &SU : DAG->SUnits) {
102 const MachineInstr *MI = SU.getInstr();
103 unsigned Op = MI->getOpcode();
104
105 if (Op == AMDGPU::ATOMIC_FENCE) {
106 // Update latency on barrier edges of ATOMIC_FENCE.
107 // Ignore scopes not expected to have any latency.
108 SyncScope::ID SSID =
109 static_cast<SyncScope::ID>(MI->getOperand(i: 1).getImm());
110 if (IgnoredScopes.contains(V: SSID))
111 continue;
112
113 for (SDep &PredDep : SU.Preds) {
114 if (!PredDep.isBarrier())
115 continue;
116 SUnit *PredSU = PredDep.getSUnit();
117 MachineInstr *MI = PredSU->getInstr();
118 // Only consider memory loads
119 if (!MI->mayLoad() || MI->mayStore())
120 continue;
121
122 addLatencyToEdge(PredDep, SU,
123 Latency: SchedModel ? SchedModel->computeInstrLatency(MI, UseDefaultDefLatency: false)
124 : FenceLatency);
125 }
126 } else if (Op == AMDGPU::S_BARRIER_WAIT) {
127 for (SDep &PredDep : SU.Preds) {
128 SUnit *PredSU = PredDep.getSUnit();
129 const MachineInstr *PredMI = PredSU->getInstr();
130 if (TII->isBarrierStart(Opcode: PredMI->getOpcode())) {
131 addLatencyToEdge(PredDep, SU, Latency: BarrierSignalWaitLatency);
132 }
133 }
134 } else if (TII->isLDSDMA(MI: *MI)) {
135 if (SIInstrFlags::usesTENSOR_CNT(O: *MI))
136 RegionTDM.push_back(Elt: &SU);
137 else if (SIInstrFlags::usesASYNC_CNT(O: *MI))
138 RegionAsync.push_back(Elt: &SU);
139 } else if (Op == AMDGPU::S_WAIT_TENSORCNT ||
140 Op == AMDGPU::S_WAIT_ASYNCCNT) {
141 auto needWaitFor = [&](SmallVectorImpl<SUnit *> &RegionLDSDMA, SUnit *SU,
142 int64_t Count) {
143 if (RegionLDSDMA.size() <= static_cast<uint64_t>(Count)) {
144 return false;
145 }
146
147 int64_t Counter = 0;
148 auto I = RegionLDSDMA.rbegin(), E = RegionLDSDMA.rend();
149 for (; I != E; I++) {
150 if (Counter >= Count)
151 return true;
152
153 if (SU->NodeNum == (*I)->NodeNum)
154 return false;
155
156 ++Counter;
157 }
158 llvm_unreachable("Malformed RegionLDSDMA");
159 };
160
161 int64_t WaitVal = MI->getOperand(i: 0).getImm();
162 for (SDep &PredDep : SU.Preds) {
163 if (PredDep.getKind() != SDep::Kind::Data)
164 continue;
165
166 Register DepReg = PredDep.getReg();
167 bool IsAsync = Op == AMDGPU::S_WAIT_ASYNCCNT;
168 Register LDSDMACnt = IsAsync ? AMDGPU::ASYNCcnt : AMDGPU::TENSORcnt;
169
170 if (DepReg != LDSDMACnt)
171 continue;
172
173 SUnit *PredSU = PredDep.getSUnit();
174
175 // The data dep can be carried by a non-LDSDMA SU
176 // (e.g. an intervening COPY or pseudo). Such predecessors are not
177 // tracked, so needWaitFor cannot reason about them.
178 const MachineInstr &PredMI = *PredSU->getInstr();
179 if (IsAsync ? !SIInstrFlags::usesASYNC_CNT(O: PredMI)
180 : !SIInstrFlags::usesTENSOR_CNT(O: PredMI))
181 continue;
182
183 if (!needWaitFor(Op == AMDGPU::S_WAIT_ASYNCCNT ? RegionAsync
184 : RegionTDM,
185 PredSU, WaitVal)) {
186 setLatencyForEdge(PredDep, SU, Latency: 1);
187 }
188 }
189 }
190 }
191}
192
193} // end namespace
194
195std::unique_ptr<ScheduleDAGMutation>
196llvm::createAMDGPUBarrierLatencyDAGMutation(MachineFunction *MF) {
197 return std::make_unique<BarrierLatency>(args&: MF);
198}
199