1//===--- AMDGPUHazardLatency.cpp - AMDGPU Hazard Latency Adjustment -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This file contains a DAG scheduling mutation to adjust the
10/// latency of data edges between instructions which use registers
11/// potentially subject to additional hazard waits not accounted
12/// for in the normal scheduling model.
13/// While the scheduling model is typically still accurate in these
14/// scenarios, adjusting latency of relevant edges can improve wait
15/// merging and reduce pipeline impact of any required waits.
16//
17//===----------------------------------------------------------------------===//
18
19#include "AMDGPUHazardLatency.h"
20#include "GCNSubtarget.h"
21#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
22#include "SIInstrInfo.h"
23#include "llvm/CodeGen/ScheduleDAGInstrs.h"
24
25using namespace llvm;
26
27namespace {
28
29class HazardLatency : public ScheduleDAGMutation {
30private:
31 const GCNSubtarget &ST;
32 const SIRegisterInfo &TRI;
33 const MachineRegisterInfo &MRI;
34
35public:
36 HazardLatency(MachineFunction *MF)
37 : ST(MF->getSubtarget<GCNSubtarget>()), TRI(*ST.getRegisterInfo()),
38 MRI(MF->getRegInfo()) {}
39 void apply(ScheduleDAGInstrs *DAG) override;
40};
41
42void HazardLatency::apply(ScheduleDAGInstrs *DAG) {
43 constexpr unsigned MaskLatencyBoost = 3;
44
45 // Hazard only manifests in Wave64
46 if (!ST.hasVALUMaskWriteHazard() || !ST.isWave64())
47 return;
48
49 for (SUnit &SU : DAG->SUnits) {
50 const MachineInstr *MI = SU.getInstr();
51 if (!SIInstrInfo::isVALU(MI: *MI))
52 continue;
53 if (MI->getOpcode() == AMDGPU::V_READLANE_B32 ||
54 MI->getOpcode() == AMDGPU::V_READFIRSTLANE_B32)
55 continue;
56 for (SDep &SuccDep : SU.Succs) {
57 if (SuccDep.isCtrl())
58 continue;
59 // Boost latency on VALU writes to SGPRs used by VALUs.
60 // Reduce risk of premature VALU pipeline stall on associated reads.
61 MachineInstr *DestMI = SuccDep.getSUnit()->getInstr();
62 if (!SIInstrInfo::isVALU(MI: *DestMI))
63 continue;
64 Register Reg = SuccDep.getReg();
65 if (!TRI.isSGPRReg(MRI, Reg))
66 continue;
67 SuccDep.setLatency(SuccDep.getLatency() * MaskLatencyBoost);
68 }
69 }
70}
71
72} // end namespace
73
74std::unique_ptr<ScheduleDAGMutation>
75llvm::createAMDGPUHazardLatencyDAGMutation(MachineFunction *MF) {
76 return std::make_unique<HazardLatency>(args&: MF);
77}
78