1//===--- AMDGPUHazardLatency.cpp - AMDGPU Hazard Latency Adjustment -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This file contains a DAG scheduling mutation to adjust the
10/// latency of data edges between instructions which use registers
11/// potentially subject to additional hazard waits not accounted
12/// for in the normal scheduling model.
13/// While the scheduling model is typically still accurate in these
14/// scenarios, adjusting latency of relevant edges can improve wait
15/// merging and reduce pipeline impact of any required waits.
16//
17//===----------------------------------------------------------------------===//
18
19#include "AMDGPUHazardLatency.h"
20#include "GCNSubtarget.h"
21#include "SIInstrInfo.h"
22#include "llvm/CodeGen/ScheduleDAGInstrs.h"
23
24using namespace llvm;
25
26namespace {
27
28class HazardLatency : public ScheduleDAGMutation {
29private:
30 const GCNSubtarget &ST;
31 const SIRegisterInfo &TRI;
32 const MachineRegisterInfo &MRI;
33
34public:
35 HazardLatency(MachineFunction *MF)
36 : ST(MF->getSubtarget<GCNSubtarget>()), TRI(*ST.getRegisterInfo()),
37 MRI(MF->getRegInfo()) {}
38 void apply(ScheduleDAGInstrs *DAG) override;
39};
40
41void HazardLatency::apply(ScheduleDAGInstrs *DAG) {
42 constexpr unsigned MaskLatencyBoost = 3;
43
44 // Hazard only manifests in Wave64
45 if (!ST.hasVALUMaskWriteHazard() || !ST.isWave64())
46 return;
47
48 for (SUnit &SU : DAG->SUnits) {
49 const MachineInstr *MI = SU.getInstr();
50 if (!SIInstrInfo::isVALU(MI: *MI, /*AllowLDSDMA=*/false))
51 continue;
52 if (MI->getOpcode() == AMDGPU::V_READLANE_B32 ||
53 MI->getOpcode() == AMDGPU::V_READFIRSTLANE_B32)
54 continue;
55 for (SDep &SuccDep : SU.Succs) {
56 if (SuccDep.isCtrl())
57 continue;
58 // Boost latency on VALU writes to SGPRs used by VALUs.
59 // Reduce risk of premature VALU pipeline stall on associated reads.
60 MachineInstr *DestMI = SuccDep.getSUnit()->getInstr();
61 if (!SIInstrInfo::isVALU(MI: *DestMI, /*AllowLDSDMA=*/false))
62 continue;
63 Register Reg = SuccDep.getReg();
64 if (!TRI.isSGPRReg(MRI, Reg))
65 continue;
66 SuccDep.setLatency(SuccDep.getLatency() * MaskLatencyBoost);
67 }
68 }
69}
70
71} // end namespace
72
73std::unique_ptr<ScheduleDAGMutation>
74llvm::createAMDGPUHazardLatencyDAGMutation(MachineFunction *MF) {
75 return std::make_unique<HazardLatency>(args&: MF);
76}
77