1//===--- AMDGPUBarrierLatency.cpp - AMDGPU Barrier Latency ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This file contains a DAG scheduling mutation to add latency to:
10/// 1. Barrier edges between ATOMIC_FENCE instructions and preceding
11/// memory accesses potentially affected by the fence.
12/// This encourages the scheduling of more instructions before
13/// ATOMIC_FENCE instructions. ATOMIC_FENCE instructions may
14/// introduce wait counting or indicate an impending S_BARRIER
15/// wait. Having more instructions in-flight across these
16/// constructs improves latency hiding.
17/// 2. Barrier edges from S_BARRIER_SIGNAL to S_BARRIER_WAIT.
18/// This encourages independent work to be scheduled between
19/// signal and wait, hiding barrier synchronization latency.
20//
21//===----------------------------------------------------------------------===//
22
23#include "AMDGPUBarrierLatency.h"
24#include "GCNSubtarget.h"
25#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
26#include "SIInstrInfo.h"
27#include "llvm/CodeGen/ScheduleDAGInstrs.h"
28#include "llvm/Support/CommandLine.h"
29#include "llvm/TargetParser/AtomicScope.h"
30
31using namespace llvm;
32
33static cl::opt<unsigned> BarrierSignalWaitLatencyOpt(
34 "amdgpu-barrier-signal-wait-latency",
35 cl::desc("Synthetic latency between S_BARRIER_SIGNAL and S_BARRIER_WAIT "
36 "to encourage scheduling independent work between them"),
37 cl::init(Val: 16), cl::Hidden);
38
39namespace {
40
41class BarrierLatency : public ScheduleDAGMutation {
42private:
43 SmallSet<SyncScope::ID, 4> IgnoredScopes;
44
45public:
46 BarrierLatency(MachineFunction *MF) {
47 LLVMContext &Context = MF->getFunction().getContext();
48 const Triple &TT = MF->getSubtarget<GCNSubtarget>().getTargetTriple();
49 auto ScopeID = [&](AtomicScope Scope, bool OneAS) {
50 return Context.getOrInsertSyncScopeID(
51 SSN: *getAtomicScopeIRString(T: TT, S: Scope, IsSingleAddressSpace: OneAS));
52 };
53 IgnoredScopes.insert(V: SyncScope::SingleThread);
54 IgnoredScopes.insert(V: ScopeID(AtomicScope::Wavefront, /*OneAS=*/false));
55 IgnoredScopes.insert(V: ScopeID(AtomicScope::Wavefront, /*OneAS=*/true));
56 IgnoredScopes.insert(V: ScopeID(AtomicScope::Single, /*OneAS=*/true));
57
58 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
59 bool TgSplit =
60 ST.hasTgSplitSupport() && AMDGPU::isTgSplitEnabled(F: MF->getFunction());
61 if (!ST.requiresWaitOnWorkgroupReleaseFence(TgSplit)) {
62 // Prior to GFX10 workgroup scope does not normally require waitcnts
63 IgnoredScopes.insert(V: ScopeID(AtomicScope::Workgroup, /*OneAS=*/false));
64 }
65 }
66 void apply(ScheduleDAGInstrs *DAG) override;
67};
68
69void addLatencyToEdge(SDep &PredDep, SUnit &SU, unsigned Latency) {
70 SUnit *PredSU = PredDep.getSUnit();
71 SDep ForwardD = PredDep;
72 ForwardD.setSUnit(&SU);
73 for (SDep &SuccDep : PredSU->Succs) {
74 if (SuccDep == ForwardD) {
75 SuccDep.setLatency(SuccDep.getLatency() + Latency);
76 break;
77 }
78 }
79 PredDep.setLatency(PredDep.getLatency() + Latency);
80 PredSU->setDepthDirty();
81 SU.setDepthDirty();
82}
83
84void setLatencyForEdge(SDep &PredDep, SUnit &SU, unsigned Latency) {
85 SUnit *PredSU = PredDep.getSUnit();
86 SDep ForwardD = PredDep;
87 ForwardD.setSUnit(&SU);
88 for (SDep &SuccDep : PredSU->Succs) {
89 if (SuccDep == ForwardD) {
90 SuccDep.setLatency(Latency);
91 break;
92 }
93 }
94 PredDep.setLatency(Latency);
95 PredSU->setDepthDirty();
96 SU.setDepthDirty();
97}
98
99void BarrierLatency::apply(ScheduleDAGInstrs *DAG) {
100 const SIInstrInfo *TII = static_cast<const SIInstrInfo *>(DAG->TII);
101 constexpr unsigned FenceLatency = 2000;
102 const unsigned BarrierSignalWaitLatency = BarrierSignalWaitLatencyOpt;
103 SmallVector<SUnit *, 8> RegionTDM;
104 SmallVector<SUnit *, 8> RegionAsync;
105 const TargetSchedModel *SchedModel = DAG->getSchedModel();
106
107 for (SUnit &SU : DAG->SUnits) {
108 const MachineInstr *MI = SU.getInstr();
109 unsigned Op = MI->getOpcode();
110
111 if (Op == AMDGPU::ATOMIC_FENCE) {
112 // Update latency on barrier edges of ATOMIC_FENCE.
113 // Ignore scopes not expected to have any latency.
114 SyncScope::ID SSID =
115 static_cast<SyncScope::ID>(MI->getOperand(i: 1).getImm());
116 if (IgnoredScopes.contains(V: SSID))
117 continue;
118
119 for (SDep &PredDep : SU.Preds) {
120 if (!PredDep.isBarrier())
121 continue;
122 SUnit *PredSU = PredDep.getSUnit();
123 MachineInstr *MI = PredSU->getInstr();
124 // Only consider memory loads
125 if (!MI->mayLoad() || MI->mayStore())
126 continue;
127
128 addLatencyToEdge(PredDep, SU,
129 Latency: SchedModel ? SchedModel->computeInstrLatency(MI, UseDefaultDefLatency: false)
130 : FenceLatency);
131 }
132 } else if (Op == AMDGPU::S_BARRIER_WAIT) {
133 for (SDep &PredDep : SU.Preds) {
134 SUnit *PredSU = PredDep.getSUnit();
135 const MachineInstr *PredMI = PredSU->getInstr();
136 if (TII->isBarrierStart(Opcode: PredMI->getOpcode())) {
137 addLatencyToEdge(PredDep, SU, Latency: BarrierSignalWaitLatency);
138 }
139 }
140 } else if (TII->isLDSDMA(MI: *MI)) {
141 if (SIInstrFlags::usesTENSOR_CNT(O: *MI))
142 RegionTDM.push_back(Elt: &SU);
143 else if (SIInstrFlags::usesASYNC_CNT(O: *MI))
144 RegionAsync.push_back(Elt: &SU);
145 } else if (Op == AMDGPU::S_WAIT_TENSORCNT ||
146 Op == AMDGPU::S_WAIT_ASYNCCNT) {
147 auto needWaitFor = [&](SmallVectorImpl<SUnit *> &RegionLDSDMA, SUnit *SU,
148 int64_t Count) {
149 if (RegionLDSDMA.size() <= static_cast<uint64_t>(Count)) {
150 return false;
151 }
152
153 int64_t Counter = 0;
154 auto I = RegionLDSDMA.rbegin(), E = RegionLDSDMA.rend();
155 for (; I != E; I++) {
156 if (Counter >= Count)
157 return true;
158
159 if (SU->NodeNum == (*I)->NodeNum)
160 return false;
161
162 ++Counter;
163 }
164 llvm_unreachable("Malformed RegionLDSDMA");
165 };
166
167 int64_t WaitVal = MI->getOperand(i: 0).getImm();
168 for (SDep &PredDep : SU.Preds) {
169 if (PredDep.getKind() != SDep::Kind::Data)
170 continue;
171
172 Register DepReg = PredDep.getReg();
173 bool IsAsync = Op == AMDGPU::S_WAIT_ASYNCCNT;
174 Register LDSDMACnt = IsAsync ? AMDGPU::ASYNCcnt : AMDGPU::TENSORcnt;
175
176 if (DepReg != LDSDMACnt)
177 continue;
178
179 SUnit *PredSU = PredDep.getSUnit();
180
181 // The data dep can be carried by a non-LDSDMA SU
182 // (e.g. an intervening COPY or pseudo). Such predecessors are not
183 // tracked, so needWaitFor cannot reason about them.
184 const MachineInstr &PredMI = *PredSU->getInstr();
185 if (IsAsync ? !SIInstrFlags::usesASYNC_CNT(O: PredMI)
186 : !SIInstrFlags::usesTENSOR_CNT(O: PredMI))
187 continue;
188
189 if (!needWaitFor(Op == AMDGPU::S_WAIT_ASYNCCNT ? RegionAsync
190 : RegionTDM,
191 PredSU, WaitVal)) {
192 setLatencyForEdge(PredDep, SU, Latency: 1);
193 }
194 }
195 }
196 }
197}
198
199} // end namespace
200
201std::unique_ptr<ScheduleDAGMutation>
202llvm::createAMDGPUBarrierLatencyDAGMutation(MachineFunction *MF) {
203 return std::make_unique<BarrierLatency>(args&: MF);
204}
205