1//===- SIInsertAsyncBarrierArriveWorkaround.cpp ---------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Add a counted replay pseudo after affected async barrier arrivals. MC
11/// emission saves EXEC, clears it for the dummy load, and restores it.
12/// Run after scheduling to keep the pseudo after its arrival, and before
13/// wait-count analysis so it accounts for the added async load.
14//
15//===----------------------------------------------------------------------===//
16
17#include "AMDGPU.h"
18#include "GCNSubtarget.h"
19#include "SIMachineFunctionInfo.h"
20#include "llvm/ADT/SmallVector.h"
21#include "llvm/CodeGen/MachineFunctionPass.h"
22#include "llvm/CodeGen/MachineInstrBuilder.h"
23#include "llvm/CodeGen/MachineRegisterInfo.h"
24#include "llvm/Support/ErrorHandling.h"
25
26using namespace llvm;
27
28#define DEBUG_TYPE "si-insert-async-barrier-arrive-workaround"
29
30static bool insertAsyncBarrierArriveWorkaround(MachineFunction &MF) {
31 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
32 if (!ST.needsAsyncBarrierArriveReplayWorkaround())
33 return false;
34
35 // Collect arrivals before inserting new definitions of ASYNCcnt.
36 SmallVector<MachineInstr *, 4> AsyncBarrierArrivals;
37 MachineRegisterInfo &MRI = MF.getRegInfo();
38 for (MachineInstr &MI : MRI.def_instructions(Reg: AMDGPU::ASYNCcnt)) {
39 if (MI.getOpcode() == AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64)
40 AsyncBarrierArrivals.push_back(Elt: &MI);
41 }
42 if (AsyncBarrierArrivals.empty())
43 return false;
44
45 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
46 const Register SavedExec = MFI->getSGPRForEXECCopy();
47 if (!SavedExec) {
48 reportFatalUsageError(
49 reason: "missing EXEC copy SGPR for async barrier workaround");
50 }
51
52 const SIInstrInfo &TII = *ST.getInstrInfo();
53 for (MachineInstr *MI : AsyncBarrierArrivals) {
54 MachineBasicBlock &MBB = *MI->getParent();
55 const DebugLoc &DL = MI->getDebugLoc();
56 auto InsertPt = std::next(x: MI->getIterator());
57 // Leave explicit S_WAIT_ASYNCCNT thresholds unchanged.
58 BuildMI(BB&: MBB, I: InsertPt, MIMD: DL,
59 MCID: TII.get(Opcode: AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8_replay))
60 .addReg(RegNo: AMDGPU::VGPR0, Flags: RegState::Undef)
61 .addReg(RegNo: AMDGPU::VGPR0_VGPR1, Flags: RegState::Undef)
62 .addImm(Val: 0)
63 .addImm(Val: 0)
64 .addReg(RegNo: SavedExec, Flags: RegState::ImplicitDefine);
65 }
66 return true;
67}
68
69namespace {
70class SIInsertAsyncBarrierArriveWorkaroundLegacy : public MachineFunctionPass {
71public:
72 static char ID;
73
74 SIInsertAsyncBarrierArriveWorkaroundLegacy() : MachineFunctionPass(ID) {}
75
76 bool runOnMachineFunction(MachineFunction &MF) override {
77 return insertAsyncBarrierArriveWorkaround(MF);
78 }
79
80 void getAnalysisUsage(AnalysisUsage &AU) const override {
81 AU.setPreservesCFG();
82 MachineFunctionPass::getAnalysisUsage(AU);
83 }
84};
85} // namespace
86
87INITIALIZE_PASS(SIInsertAsyncBarrierArriveWorkaroundLegacy, DEBUG_TYPE,
88 "SI insert async barrier arrival workaround", false, false)
89
90char SIInsertAsyncBarrierArriveWorkaroundLegacy::ID = 0;
91
92FunctionPass *llvm::createSIInsertAsyncBarrierArriveWorkaroundPass() {
93 return new SIInsertAsyncBarrierArriveWorkaroundLegacy();
94}
95
96PreservedAnalyses SIInsertAsyncBarrierArriveWorkaroundPass::run(
97 MachineFunction &MF, MachineFunctionAnalysisManager &MFAM) {
98 if (!insertAsyncBarrierArriveWorkaround(MF))
99 return PreservedAnalyses::all();
100 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
101 PA.preserveSet<CFGAnalyses>();
102 return PA;
103}
104