| 1 | //===- SIInsertAsyncBarrierArriveWorkaround.cpp ---------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | /// \file |
| 10 | /// Add a counted replay pseudo after affected async barrier arrivals. MC |
| 11 | /// emission saves EXEC, clears it for the dummy load, and restores it. |
| 12 | /// Run after scheduling to keep the pseudo after its arrival, and before |
| 13 | /// wait-count analysis so it accounts for the added async load. |
| 14 | // |
| 15 | //===----------------------------------------------------------------------===// |
| 16 | |
| 17 | #include "AMDGPU.h" |
| 18 | #include "GCNSubtarget.h" |
| 19 | #include "SIMachineFunctionInfo.h" |
| 20 | #include "llvm/ADT/SmallVector.h" |
| 21 | #include "llvm/CodeGen/MachineFunctionPass.h" |
| 22 | #include "llvm/CodeGen/MachineInstrBuilder.h" |
| 23 | #include "llvm/CodeGen/MachineRegisterInfo.h" |
| 24 | #include "llvm/Support/ErrorHandling.h" |
| 25 | |
| 26 | using namespace llvm; |
| 27 | |
| 28 | #define DEBUG_TYPE "si-insert-async-barrier-arrive-workaround" |
| 29 | |
| 30 | static bool insertAsyncBarrierArriveWorkaround(MachineFunction &MF) { |
| 31 | const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); |
| 32 | if (!ST.needsAsyncBarrierArriveReplayWorkaround()) |
| 33 | return false; |
| 34 | |
| 35 | // Collect arrivals before inserting new definitions of ASYNCcnt. |
| 36 | SmallVector<MachineInstr *, 4> AsyncBarrierArrivals; |
| 37 | MachineRegisterInfo &MRI = MF.getRegInfo(); |
| 38 | for (MachineInstr &MI : MRI.def_instructions(Reg: AMDGPU::ASYNCcnt)) { |
| 39 | if (MI.getOpcode() == AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64) |
| 40 | AsyncBarrierArrivals.push_back(Elt: &MI); |
| 41 | } |
| 42 | if (AsyncBarrierArrivals.empty()) |
| 43 | return false; |
| 44 | |
| 45 | const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); |
| 46 | const Register SavedExec = MFI->getSGPRForEXECCopy(); |
| 47 | if (!SavedExec) { |
| 48 | reportFatalUsageError( |
| 49 | reason: "missing EXEC copy SGPR for async barrier workaround" ); |
| 50 | } |
| 51 | |
| 52 | const SIInstrInfo &TII = *ST.getInstrInfo(); |
| 53 | for (MachineInstr *MI : AsyncBarrierArrivals) { |
| 54 | MachineBasicBlock &MBB = *MI->getParent(); |
| 55 | const DebugLoc &DL = MI->getDebugLoc(); |
| 56 | auto InsertPt = std::next(x: MI->getIterator()); |
| 57 | // Leave explicit S_WAIT_ASYNCCNT thresholds unchanged. |
| 58 | BuildMI(BB&: MBB, I: InsertPt, MIMD: DL, |
| 59 | MCID: TII.get(Opcode: AMDGPU::GLOBAL_LOAD_ASYNC_TO_LDS_B8_replay)) |
| 60 | .addReg(RegNo: AMDGPU::VGPR0, Flags: RegState::Undef) |
| 61 | .addReg(RegNo: AMDGPU::VGPR0_VGPR1, Flags: RegState::Undef) |
| 62 | .addImm(Val: 0) |
| 63 | .addImm(Val: 0) |
| 64 | .addReg(RegNo: SavedExec, Flags: RegState::ImplicitDefine); |
| 65 | } |
| 66 | return true; |
| 67 | } |
| 68 | |
| 69 | namespace { |
| 70 | class SIInsertAsyncBarrierArriveWorkaroundLegacy : public MachineFunctionPass { |
| 71 | public: |
| 72 | static char ID; |
| 73 | |
| 74 | SIInsertAsyncBarrierArriveWorkaroundLegacy() : MachineFunctionPass(ID) {} |
| 75 | |
| 76 | bool runOnMachineFunction(MachineFunction &MF) override { |
| 77 | return insertAsyncBarrierArriveWorkaround(MF); |
| 78 | } |
| 79 | |
| 80 | void getAnalysisUsage(AnalysisUsage &AU) const override { |
| 81 | AU.setPreservesCFG(); |
| 82 | MachineFunctionPass::getAnalysisUsage(AU); |
| 83 | } |
| 84 | }; |
| 85 | } // namespace |
| 86 | |
| 87 | INITIALIZE_PASS(SIInsertAsyncBarrierArriveWorkaroundLegacy, DEBUG_TYPE, |
| 88 | "SI insert async barrier arrival workaround" , false, false) |
| 89 | |
| 90 | char SIInsertAsyncBarrierArriveWorkaroundLegacy::ID = 0; |
| 91 | |
| 92 | FunctionPass *llvm::createSIInsertAsyncBarrierArriveWorkaroundPass() { |
| 93 | return new SIInsertAsyncBarrierArriveWorkaroundLegacy(); |
| 94 | } |
| 95 | |
| 96 | PreservedAnalyses SIInsertAsyncBarrierArriveWorkaroundPass::run( |
| 97 | MachineFunction &MF, MachineFunctionAnalysisManager &MFAM) { |
| 98 | if (!insertAsyncBarrierArriveWorkaround(MF)) |
| 99 | return PreservedAnalyses::all(); |
| 100 | PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses(); |
| 101 | PA.preserveSet<CFGAnalyses>(); |
| 102 | return PA; |
| 103 | } |
| 104 | |