1//==- SIMachineFunctionInfo.h - SIMachineFunctionInfo interface --*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_LIB_TARGET_AMDGPU_SIMACHINEFUNCTIONINFO_H
14#define LLVM_LIB_TARGET_AMDGPU_SIMACHINEFUNCTIONINFO_H
15
16#include "AMDGPUArgumentUsageInfo.h"
17#include "AMDGPUMachineFunctionInfo.h"
18#include "AMDGPUTargetMachine.h"
19#include "GCNSubtarget.h"
20#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
21#include "SIInstrInfo.h"
22#include "SIModeRegisterDefaults.h"
23#include "llvm/ADT/SetVector.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/CodeGen/MIRYamlMapping.h"
26#include "llvm/CodeGen/PseudoSourceValue.h"
27#include "llvm/Support/raw_ostream.h"
28#include <optional>
29
30namespace llvm {
31
32class MachineFrameInfo;
33class MachineFunction;
34class SIMachineFunctionInfo;
35class SIRegisterInfo;
36class MCRegisterClass;
37using TargetRegisterClass = MCRegisterClass;
38
39class AMDGPUPseudoSourceValue : public PseudoSourceValue {
40public:
41 enum AMDGPUPSVKind : unsigned {
42 PSVImage = PseudoSourceValue::TargetCustom,
43 GWSResource
44 };
45
46protected:
47 AMDGPUPseudoSourceValue(unsigned Kind, const AMDGPUTargetMachine &TM)
48 : PseudoSourceValue(Kind, TM) {}
49
50public:
51 bool isConstant(const MachineFrameInfo *) const override {
52 // This should probably be true for most images, but we will start by being
53 // conservative.
54 return false;
55 }
56
57 bool isAliased(const MachineFrameInfo *) const override {
58 return true;
59 }
60
61 bool mayAlias(const MachineFrameInfo *) const override {
62 return true;
63 }
64};
65
66class AMDGPUGWSResourcePseudoSourceValue final : public AMDGPUPseudoSourceValue {
67public:
68 explicit AMDGPUGWSResourcePseudoSourceValue(const AMDGPUTargetMachine &TM)
69 : AMDGPUPseudoSourceValue(GWSResource, TM) {}
70
71 static bool classof(const PseudoSourceValue *V) {
72 return V->kind() == GWSResource;
73 }
74
75 // These are inaccessible memory from IR.
76 bool isAliased(const MachineFrameInfo *) const override {
77 return false;
78 }
79
80 // These are inaccessible memory from IR.
81 bool mayAlias(const MachineFrameInfo *) const override {
82 return false;
83 }
84
85 void printCustom(raw_ostream &OS) const override {
86 OS << "GWSResource";
87 }
88};
89
90namespace yaml {
91
92struct SIArgument {
93 bool IsRegister;
94 union {
95 StringValue RegisterName;
96 unsigned StackOffset;
97 };
98 std::optional<unsigned> Mask;
99
100 // Default constructor, which creates a stack argument.
101 SIArgument() : IsRegister(false), StackOffset(0) {}
102 SIArgument(const SIArgument &Other) {
103 IsRegister = Other.IsRegister;
104 if (IsRegister)
105 new (&RegisterName) StringValue(Other.RegisterName);
106 else
107 StackOffset = Other.StackOffset;
108 Mask = Other.Mask;
109 }
110 SIArgument &operator=(const SIArgument &Other) {
111 // Default-construct or destruct the old RegisterName in case of switching
112 // union members
113 if (IsRegister != Other.IsRegister) {
114 if (Other.IsRegister)
115 new (&RegisterName) StringValue();
116 else
117 RegisterName.~StringValue();
118 }
119 IsRegister = Other.IsRegister;
120 if (IsRegister)
121 RegisterName = Other.RegisterName;
122 else
123 StackOffset = Other.StackOffset;
124 Mask = Other.Mask;
125 return *this;
126 }
127 ~SIArgument() {
128 if (IsRegister)
129 RegisterName.~StringValue();
130 }
131
132 // Helper to create a register or stack argument.
133 static inline SIArgument createArgument(bool IsReg) {
134 if (IsReg)
135 return SIArgument(IsReg);
136 return SIArgument();
137 }
138
139private:
140 // Construct a register argument.
141 SIArgument(bool) : IsRegister(true), RegisterName() {}
142};
143
144template <> struct MappingTraits<SIArgument> {
145 static void mapping(IO &YamlIO, SIArgument &A) {
146 if (YamlIO.outputting()) {
147 if (A.IsRegister)
148 YamlIO.mapRequired(Key: "reg", Val&: A.RegisterName);
149 else
150 YamlIO.mapRequired(Key: "offset", Val&: A.StackOffset);
151 } else {
152 auto Keys = YamlIO.keys();
153 if (is_contained(Range&: Keys, Element: "reg")) {
154 A = SIArgument::createArgument(IsReg: true);
155 YamlIO.mapRequired(Key: "reg", Val&: A.RegisterName);
156 } else if (is_contained(Range&: Keys, Element: "offset"))
157 YamlIO.mapRequired(Key: "offset", Val&: A.StackOffset);
158 else
159 YamlIO.setError("missing required key 'reg' or 'offset'");
160 }
161 YamlIO.mapOptional(Key: "mask", Val&: A.Mask);
162 }
163 static const bool flow = true;
164};
165
166struct SIArgumentInfo {
167 std::optional<SIArgument> PrivateSegmentBuffer;
168 std::optional<SIArgument> DispatchPtr;
169 std::optional<SIArgument> QueuePtr;
170 std::optional<SIArgument> KernargSegmentPtr;
171 std::optional<SIArgument> DispatchID;
172 std::optional<SIArgument> FlatScratchInit;
173 std::optional<SIArgument> PrivateSegmentSize;
174 std::optional<SIArgument> FirstKernArgPreloadReg;
175
176 std::optional<SIArgument> WorkGroupIDX;
177 std::optional<SIArgument> WorkGroupIDY;
178 std::optional<SIArgument> WorkGroupIDZ;
179 std::optional<SIArgument> WorkGroupInfo;
180 std::optional<SIArgument> LDSKernelId;
181 std::optional<SIArgument> PrivateSegmentWaveByteOffset;
182
183 std::optional<SIArgument> ImplicitArgPtr;
184 std::optional<SIArgument> ImplicitBufferPtr;
185
186 std::optional<SIArgument> WorkItemIDX;
187 std::optional<SIArgument> WorkItemIDY;
188 std::optional<SIArgument> WorkItemIDZ;
189};
190
191template <> struct MappingTraits<SIArgumentInfo> {
192 static void mapping(IO &YamlIO, SIArgumentInfo &AI) {
193 YamlIO.mapOptional(Key: "privateSegmentBuffer", Val&: AI.PrivateSegmentBuffer);
194 YamlIO.mapOptional(Key: "dispatchPtr", Val&: AI.DispatchPtr);
195 YamlIO.mapOptional(Key: "queuePtr", Val&: AI.QueuePtr);
196 YamlIO.mapOptional(Key: "kernargSegmentPtr", Val&: AI.KernargSegmentPtr);
197 YamlIO.mapOptional(Key: "dispatchID", Val&: AI.DispatchID);
198 YamlIO.mapOptional(Key: "flatScratchInit", Val&: AI.FlatScratchInit);
199 YamlIO.mapOptional(Key: "privateSegmentSize", Val&: AI.PrivateSegmentSize);
200 YamlIO.mapOptional(Key: "firstKernArgPreloadReg", Val&: AI.FirstKernArgPreloadReg);
201
202 YamlIO.mapOptional(Key: "workGroupIDX", Val&: AI.WorkGroupIDX);
203 YamlIO.mapOptional(Key: "workGroupIDY", Val&: AI.WorkGroupIDY);
204 YamlIO.mapOptional(Key: "workGroupIDZ", Val&: AI.WorkGroupIDZ);
205 YamlIO.mapOptional(Key: "workGroupInfo", Val&: AI.WorkGroupInfo);
206 YamlIO.mapOptional(Key: "LDSKernelId", Val&: AI.LDSKernelId);
207 YamlIO.mapOptional(Key: "privateSegmentWaveByteOffset",
208 Val&: AI.PrivateSegmentWaveByteOffset);
209
210 YamlIO.mapOptional(Key: "implicitArgPtr", Val&: AI.ImplicitArgPtr);
211 YamlIO.mapOptional(Key: "implicitBufferPtr", Val&: AI.ImplicitBufferPtr);
212
213 YamlIO.mapOptional(Key: "workItemIDX", Val&: AI.WorkItemIDX);
214 YamlIO.mapOptional(Key: "workItemIDY", Val&: AI.WorkItemIDY);
215 YamlIO.mapOptional(Key: "workItemIDZ", Val&: AI.WorkItemIDZ);
216 }
217};
218
219// Default to default mode for default calling convention.
220struct SIMode {
221 bool IEEE = true;
222 bool DX10Clamp = true;
223 bool FP32InputDenormals = true;
224 bool FP32OutputDenormals = true;
225 bool FP64FP16InputDenormals = true;
226 bool FP64FP16OutputDenormals = true;
227
228 SIMode() = default;
229
230 SIMode(const SIModeRegisterDefaults &Mode) {
231 IEEE = Mode.IEEE;
232 DX10Clamp = Mode.DX10Clamp;
233 FP32InputDenormals = Mode.FP32Denormals.Input != DenormalMode::PreserveSign;
234 FP32OutputDenormals =
235 Mode.FP32Denormals.Output != DenormalMode::PreserveSign;
236 FP64FP16InputDenormals =
237 Mode.FP64FP16Denormals.Input != DenormalMode::PreserveSign;
238 FP64FP16OutputDenormals =
239 Mode.FP64FP16Denormals.Output != DenormalMode::PreserveSign;
240 }
241
242 bool operator ==(const SIMode Other) const {
243 return IEEE == Other.IEEE &&
244 DX10Clamp == Other.DX10Clamp &&
245 FP32InputDenormals == Other.FP32InputDenormals &&
246 FP32OutputDenormals == Other.FP32OutputDenormals &&
247 FP64FP16InputDenormals == Other.FP64FP16InputDenormals &&
248 FP64FP16OutputDenormals == Other.FP64FP16OutputDenormals;
249 }
250};
251
252template <> struct MappingTraits<SIMode> {
253 static void mapping(IO &YamlIO, SIMode &Mode) {
254 YamlIO.mapOptional(Key: "ieee", Val&: Mode.IEEE, Default: true);
255 YamlIO.mapOptional(Key: "dx10-clamp", Val&: Mode.DX10Clamp, Default: true);
256 YamlIO.mapOptional(Key: "fp32-input-denormals", Val&: Mode.FP32InputDenormals, Default: true);
257 YamlIO.mapOptional(Key: "fp32-output-denormals", Val&: Mode.FP32OutputDenormals, Default: true);
258 YamlIO.mapOptional(Key: "fp64-fp16-input-denormals", Val&: Mode.FP64FP16InputDenormals, Default: true);
259 YamlIO.mapOptional(Key: "fp64-fp16-output-denormals", Val&: Mode.FP64FP16OutputDenormals, Default: true);
260 }
261};
262
263struct SIMachineFunctionInfo final : public yaml::MachineFunctionInfo {
264 uint64_t ExplicitKernArgSize = 0;
265 Align MaxKernArgAlign;
266 uint32_t LDSSize = 0;
267 uint32_t GDSSize = 0;
268 Align DynLDSAlign;
269 bool IsEntryFunction = false;
270 bool IsChainFunction = false;
271 bool MemoryBound = false;
272 bool WaveLimiter = false;
273 bool HasSpilledSGPRs = false;
274 bool HasSpilledVGPRs = false;
275 bool HasNoWWMPoolSGPRSpillFallback = false;
276 uint16_t NumWaveDispatchSGPRs = 0;
277 uint16_t NumWaveDispatchVGPRs = 0;
278 uint32_t HighBitsOf32BitAddress = 0;
279
280 // TODO: 10 may be a better default since it's the maximum.
281 unsigned Occupancy = 0;
282
283 SmallVector<StringValue, 2> SpillPhysVGPRS;
284 SmallVector<StringValue> WWMReservedRegs;
285
286 StringValue ScratchRSrcReg = "$private_rsrc_reg";
287 StringValue FrameOffsetReg = "$fp_reg";
288 StringValue StackPtrOffsetReg = "$sp_reg";
289
290 unsigned BytesInStackArgArea = 0;
291 bool ReturnsVoid = true;
292
293 std::optional<SIArgumentInfo> ArgInfo;
294
295 unsigned PSInputAddr = 0;
296 unsigned PSInputEnable = 0;
297 unsigned MaxMemoryClusterDWords = DefaultMemoryClusterDWordsLimit;
298
299 SIMode Mode;
300 std::optional<FrameIndex> ScavengeFI;
301 StringValue VGPRForAGPRCopy;
302 StringValue SGPRForEXECCopy;
303 StringValue LongBranchReservedReg;
304
305 bool HasInitWholeWave = false;
306 bool IsWholeWaveFunction = false;
307
308 std::optional<unsigned> DynamicVGPRBlockSize;
309 unsigned ScratchReservedForDynamicVGPRs = 0;
310
311 unsigned NumKernargPreloadSGPRs = 0;
312
313 unsigned MinNumAGPRs = ~0u;
314
315 SIMachineFunctionInfo() = default;
316 SIMachineFunctionInfo(const llvm::SIMachineFunctionInfo &,
317 const TargetRegisterInfo &TRI,
318 const llvm::MachineFunction &MF);
319
320 void mappingImpl(yaml::IO &YamlIO) override;
321 ~SIMachineFunctionInfo() override = default;
322};
323
324template <> struct MappingTraits<SIMachineFunctionInfo> {
325 static void mapping(IO &YamlIO, SIMachineFunctionInfo &MFI) {
326 YamlIO.mapOptional(Key: "explicitKernArgSize", Val&: MFI.ExplicitKernArgSize,
327 UINT64_C(0));
328 YamlIO.mapOptional(Key: "maxKernArgAlign", Val&: MFI.MaxKernArgAlign);
329 YamlIO.mapOptional(Key: "ldsSize", Val&: MFI.LDSSize, Default: 0u);
330 YamlIO.mapOptional(Key: "gdsSize", Val&: MFI.GDSSize, Default: 0u);
331 YamlIO.mapOptional(Key: "dynLDSAlign", Val&: MFI.DynLDSAlign, Default: Align());
332 YamlIO.mapOptional(Key: "isEntryFunction", Val&: MFI.IsEntryFunction, Default: false);
333 YamlIO.mapOptional(Key: "isChainFunction", Val&: MFI.IsChainFunction, Default: false);
334 YamlIO.mapOptional(Key: "memoryBound", Val&: MFI.MemoryBound, Default: false);
335 YamlIO.mapOptional(Key: "waveLimiter", Val&: MFI.WaveLimiter, Default: false);
336 YamlIO.mapOptional(Key: "hasSpilledSGPRs", Val&: MFI.HasSpilledSGPRs, Default: false);
337 YamlIO.mapOptional(Key: "hasSpilledVGPRs", Val&: MFI.HasSpilledVGPRs, Default: false);
338 YamlIO.mapOptional(Key: "hasNoWWMPoolSGPRSpillFallback",
339 Val&: MFI.HasNoWWMPoolSGPRSpillFallback, Default: false);
340 YamlIO.mapOptional(Key: "numWaveDispatchSGPRs", Val&: MFI.NumWaveDispatchSGPRs, Default: false);
341 YamlIO.mapOptional(Key: "numWaveDispatchVGPRs", Val&: MFI.NumWaveDispatchVGPRs, Default: false);
342 YamlIO.mapOptional(Key: "scratchRSrcReg", Val&: MFI.ScratchRSrcReg,
343 Default: StringValue("$private_rsrc_reg"));
344 YamlIO.mapOptional(Key: "frameOffsetReg", Val&: MFI.FrameOffsetReg,
345 Default: StringValue("$fp_reg"));
346 YamlIO.mapOptional(Key: "stackPtrOffsetReg", Val&: MFI.StackPtrOffsetReg,
347 Default: StringValue("$sp_reg"));
348 YamlIO.mapOptional(Key: "bytesInStackArgArea", Val&: MFI.BytesInStackArgArea, Default: 0u);
349 YamlIO.mapOptional(Key: "returnsVoid", Val&: MFI.ReturnsVoid, Default: true);
350 YamlIO.mapOptional(Key: "argumentInfo", Val&: MFI.ArgInfo);
351 YamlIO.mapOptional(Key: "psInputAddr", Val&: MFI.PSInputAddr, Default: 0u);
352 YamlIO.mapOptional(Key: "psInputEnable", Val&: MFI.PSInputEnable, Default: 0u);
353 YamlIO.mapOptional(Key: "maxMemoryClusterDWords", Val&: MFI.MaxMemoryClusterDWords,
354 Default: DefaultMemoryClusterDWordsLimit);
355 YamlIO.mapOptional(Key: "mode", Val&: MFI.Mode, Default: SIMode());
356 YamlIO.mapOptional(Key: "highBitsOf32BitAddress",
357 Val&: MFI.HighBitsOf32BitAddress, Default: 0u);
358 YamlIO.mapOptional(Key: "occupancy", Val&: MFI.Occupancy, Default: 0);
359 YamlIO.mapOptional(Key: "spillPhysVGPRs", Val&: MFI.SpillPhysVGPRS);
360 YamlIO.mapOptional(Key: "wwmReservedRegs", Val&: MFI.WWMReservedRegs);
361 YamlIO.mapOptional(Key: "scavengeFI", Val&: MFI.ScavengeFI);
362 YamlIO.mapOptional(Key: "vgprForAGPRCopy", Val&: MFI.VGPRForAGPRCopy,
363 Default: StringValue()); // Don't print out when it's empty.
364 YamlIO.mapOptional(Key: "sgprForEXECCopy", Val&: MFI.SGPRForEXECCopy,
365 Default: StringValue()); // Don't print out when it's empty.
366 YamlIO.mapOptional(Key: "longBranchReservedReg", Val&: MFI.LongBranchReservedReg,
367 Default: StringValue());
368 YamlIO.mapOptional(Key: "hasInitWholeWave", Val&: MFI.HasInitWholeWave, Default: false);
369 YamlIO.mapOptional(Key: "dynamicVGPRBlockSize", Val&: MFI.DynamicVGPRBlockSize);
370 YamlIO.mapOptional(Key: "scratchReservedForDynamicVGPRs",
371 Val&: MFI.ScratchReservedForDynamicVGPRs, Default: 0);
372 YamlIO.mapOptional(Key: "numKernargPreloadSGPRs", Val&: MFI.NumKernargPreloadSGPRs, Default: 0);
373 YamlIO.mapOptional(Key: "isWholeWaveFunction", Val&: MFI.IsWholeWaveFunction, Default: false);
374 YamlIO.mapOptional(Key: "minNumAGPRs", Val&: MFI.MinNumAGPRs, Default: ~0u);
375 }
376};
377
378} // end namespace yaml
379
380// A CSR SGPR value can be preserved inside a callee using one of the following
381// methods.
382// 1. Copy to an unused scratch SGPR.
383// 2. Spill to a VGPR lane.
384// 3. Spill to memory via. a scratch VGPR.
385// class PrologEpilogSGPRSaveRestoreInfo represents the save/restore method used
386// for an SGPR at function prolog/epilog.
387enum class SGPRSaveKind : uint8_t {
388 COPY_TO_SCRATCH_SGPR,
389 SPILL_TO_VGPR_LANE,
390 SPILL_TO_MEM
391};
392
393class PrologEpilogSGPRSaveRestoreInfo {
394 SGPRSaveKind Kind;
395 union {
396 int Index;
397 Register Reg;
398 };
399
400public:
401 PrologEpilogSGPRSaveRestoreInfo(SGPRSaveKind K, int I) : Kind(K), Index(I) {}
402 PrologEpilogSGPRSaveRestoreInfo(SGPRSaveKind K, Register R)
403 : Kind(K), Reg(R) {}
404 Register getReg() const { return Reg; }
405 int getIndex() const { return Index; }
406 SGPRSaveKind getKind() const { return Kind; }
407};
408
409struct VGPRBlock2IndexFunctor {
410 using argument_type = Register;
411 unsigned operator()(Register Reg) const {
412 assert(AMDGPU::VReg_1024RegClass.contains(Reg) && "Expecting a VGPR block");
413
414 const MCRegister FirstVGPRBlock = AMDGPU::VReg_1024RegClass.getRegister(i: 0);
415 return Reg - FirstVGPRBlock;
416 }
417};
418
419/// This class keeps track of the SPI_SP_INPUT_ADDR config register, which
420/// tells the hardware which interpolation parameters to load.
421class SIMachineFunctionInfo final : public AMDGPUMachineFunctionInfo,
422 private MachineRegisterInfo::Delegate {
423 friend class GCNTargetMachine;
424
425 // State of MODE register, assumed FP mode.
426 SIModeRegisterDefaults Mode;
427
428 // Registers that may be reserved for spilling purposes. These may be the same
429 // as the input registers.
430 Register ScratchRSrcReg = AMDGPU::PRIVATE_RSRC_REG;
431
432 // This is the unswizzled offset from the current dispatch's scratch wave
433 // base to the beginning of the current function's frame.
434 Register FrameOffsetReg = AMDGPU::FP_REG;
435
436 // This is an ABI register used in the non-entry calling convention to
437 // communicate the unswizzled offset from the current dispatch's scratch wave
438 // base to the beginning of the new function's frame.
439 Register StackPtrOffsetReg = AMDGPU::SP_REG;
440
441 // Registers that may be reserved when RA doesn't allocate enough
442 // registers to plan for the case where an indirect branch ends up
443 // being needed during branch relaxation.
444 Register LongBranchReservedReg;
445
446 AMDGPUFunctionArgInfo ArgInfo;
447
448 // Graphics info.
449 unsigned PSInputAddr = 0;
450 unsigned PSInputEnable = 0;
451
452 /// Number of bytes of arguments this function has on the stack. If the callee
453 /// is expected to restore the argument stack this should be a multiple of 16,
454 /// all usable during a tail call.
455 ///
456 /// The alternative would forbid tail call optimisation in some cases: if we
457 /// want to transfer control from a function with 8-bytes of stack-argument
458 /// space to a function with 16-bytes then misalignment of this value would
459 /// make a stack adjustment necessary, which could not be undone by the
460 /// callee.
461 unsigned BytesInStackArgArea = 0;
462
463 bool ReturnsVoid = true;
464
465 // A pair of default/requested minimum/maximum flat work group sizes.
466 // Minimum - first, maximum - second.
467 std::pair<unsigned, unsigned> FlatWorkGroupSizes = {0, 0};
468
469 // A pair of default/requested minimum/maximum number of waves per execution
470 // unit. Minimum - first, maximum - second.
471 std::pair<unsigned, unsigned> WavesPerEU = {0, 0};
472
473 const AMDGPUGWSResourcePseudoSourceValue GWSResourcePSV;
474
475 // Default/requested number of work groups for the function.
476 SmallVector<unsigned> MaxNumWorkGroups = {0, 0, 0};
477
478 // Requested cluster dimensions.
479 AMDGPU::ClusterDimsAttr ClusterDims;
480
481private:
482 unsigned NumUserSGPRs = 0;
483 unsigned NumSystemSGPRs = 0;
484
485 unsigned NumWaveDispatchSGPRs = 0;
486 unsigned NumWaveDispatchVGPRs = 0;
487
488 bool HasSpilledSGPRs = false;
489 bool HasSpilledVGPRs = false;
490 bool HasNonSpillStackObjects = false;
491 bool IsStackRealigned = false;
492
493 unsigned NumSpilledSGPRs = 0;
494 unsigned NumSpilledVGPRs = 0;
495
496 unsigned DynamicVGPRBlockSize = 0;
497
498 // The size in bytes of the scratch space reserved for the CWSR trap handler
499 // to spill some of the dynamic VGPRs.
500 unsigned ScratchReservedForDynamicVGPRs = 0;
501
502 // Tracks information about user SGPRs that will be setup by hardware which
503 // will apply to all wavefronts of the grid.
504 GCNUserSGPRUsageInfo UserSGPRInfo;
505
506 // Feature bits required for inputs passed in system SGPRs.
507 bool WorkGroupIDX : 1; // Always initialized.
508 bool WorkGroupIDY : 1;
509 bool WorkGroupIDZ : 1;
510 bool WorkGroupInfo : 1;
511 bool LDSKernelId : 1;
512 bool PrivateSegmentWaveByteOffset : 1;
513
514 bool WorkItemIDX : 1; // Always initialized.
515 bool WorkItemIDY : 1;
516 bool WorkItemIDZ : 1;
517
518 // Pointer to where the ABI inserts special kernel arguments separate from the
519 // user arguments. This is an offset from the KernargSegmentPtr.
520 bool ImplicitArgPtr : 1;
521
522 /// Minimum number of AGPRs required to allocate in the function. Only
523 /// relevant for gfx90a-gfx950. For gfx908, this should be infinite.
524 unsigned MinNumAGPRs = ~0u;
525
526 // The hard-wired high half of the address of the global information table
527 // for AMDPAL OS type. 0xffffffff represents no hard-wired high half, since
528 // current hardware only allows a 16 bit value.
529 unsigned GITPtrHigh;
530
531 unsigned HighBitsOf32BitAddress;
532
533 // Flags associated with the virtual registers.
534 IndexedMap<uint8_t, VirtReg2IndexFunctor> VRegFlags;
535
536 // Current recorded maximum possible occupancy.
537 unsigned Occupancy;
538
539 // Maximum number of dwords that can be clusterred during instruction
540 // scheduler stage.
541 unsigned MaxMemoryClusterDWords = DefaultMemoryClusterDWordsLimit;
542
543 MCPhysReg getNextUserSGPR() const;
544
545 MCPhysReg getNextSystemSGPR() const;
546
547 // MachineRegisterInfo callback functions to notify events.
548 void MRI_NoteNewVirtualRegister(Register Reg) override;
549 void MRI_NoteCloneVirtualRegister(Register NewReg, Register SrcReg) override;
550
551public:
552 static bool MFMAVGPRForm;
553
554 struct VGPRSpillToAGPR {
555 SmallVector<MCPhysReg, 32> Lanes;
556 bool FullyAllocated = false;
557 bool IsDead = false;
558 };
559
560private:
561 // To track virtual VGPR + lane index for each subregister of the SGPR spilled
562 // to frameindex key during SILowerSGPRSpills pass.
563 DenseMap<int, std::vector<SIRegisterInfo::SpilledReg>>
564 SGPRSpillsToVirtualVGPRLanes;
565 // To track physical VGPR + lane index for CSR SGPR spills and special SGPRs
566 // like Frame Pointer identified during PrologEpilogInserter.
567 DenseMap<int, std::vector<SIRegisterInfo::SpilledReg>>
568 SGPRSpillsToPhysicalVGPRLanes;
569 unsigned NumVirtualVGPRSpillLanes = 0;
570 unsigned NumPhysicalVGPRSpillLanes = 0;
571 SmallVector<Register, 2> SpillVGPRs;
572 SmallVector<Register, 2> SpillPhysVGPRs;
573 using WWMSpillsMap = MapVector<Register, int>;
574 // To track the registers used in instructions that can potentially modify the
575 // inactive lanes. The WWM instructions and the writelane instructions for
576 // spilling SGPRs to VGPRs fall under such category of operations. The VGPRs
577 // modified by them should be spilled/restored at function prolog/epilog to
578 // avoid any undesired outcome. Each entry in this map holds a pair of values,
579 // the VGPR and its stack slot index.
580 WWMSpillsMap WWMSpills;
581
582 // Before allocation, the VGPR registers are partitioned into two distinct
583 // sets, the first one for WWM values and the second set for per-lane values.
584 // The latter set should be reserved during WWM-regalloc.
585 BitVector PerLaneVGPRMask;
586
587 using ReservedRegSet = SmallSetVector<Register, 8>;
588 // To track the VGPRs reserved for WWM instructions. They get stack slots
589 // later during PrologEpilogInserter and get added into the superset WWMSpills
590 // for actual spilling. A separate set makes the register reserved part and
591 // the serialization easier.
592 ReservedRegSet WWMReservedRegs;
593
594 bool IsWholeWaveFunction = false;
595
596 using PrologEpilogSGPRSpill =
597 std::pair<Register, PrologEpilogSGPRSaveRestoreInfo>;
598 // To track the SGPR spill method used for a CSR SGPR register during
599 // frame lowering. Even though the SGPR spills are handled during
600 // SILowerSGPRSpills pass, some special handling needed later during the
601 // PrologEpilogInserter.
602 SmallVector<PrologEpilogSGPRSpill, 3> PrologEpilogSGPRSpills;
603
604 // To save/restore EXEC MASK around WWM spills and copies.
605 Register SGPRForEXECCopy;
606
607 DenseMap<int, VGPRSpillToAGPR> VGPRToAGPRSpills;
608
609 // AGPRs used for VGPR spills.
610 SmallVector<MCPhysReg, 32> SpillAGPR;
611
612 // VGPRs used for AGPR spills.
613 SmallVector<MCPhysReg, 32> SpillVGPR;
614
615 // Emergency stack slot. Sometimes, we create this before finalizing the stack
616 // frame, so save it here and add it to the RegScavenger later.
617 std::optional<int> ScavengeFI;
618
619 // Ordinary SGPR spills fell back to memory because no WWM VGPR pool was
620 // available. This path may require additional nested VGPR scavenging slots
621 // during frame-index elimination.
622 bool HasNoWWMPoolSGPRSpillFallback = false;
623
624 // Map each VGPR CSR to the mask needed to save and restore it using block
625 // load/store instructions. Only used if the subtarget feature for VGPR block
626 // load/store is enabled.
627 IndexedMap<uint32_t, VGPRBlock2IndexFunctor> MaskForVGPRBlockOps;
628
629private:
630 Register VGPRForAGPRCopy;
631
632 bool allocateVirtualVGPRForSGPRSpills(MachineFunction &MF, int FI,
633 unsigned LaneIndex);
634 bool allocatePhysicalVGPRForSGPRSpills(MachineFunction &MF, int FI,
635 unsigned LaneIndex,
636 bool IsPrologEpilog);
637
638public:
639 Register getVGPRForAGPRCopy() const {
640 return VGPRForAGPRCopy;
641 }
642
643 void setVGPRForAGPRCopy(Register NewVGPRForAGPRCopy) {
644 VGPRForAGPRCopy = NewVGPRForAGPRCopy;
645 }
646
647 bool isCalleeSavedReg(const MCPhysReg *CSRegs, MCPhysReg Reg) const;
648
649 void setMaskForVGPRBlockOps(Register RegisterBlock, uint32_t Mask) {
650 MaskForVGPRBlockOps.grow(N: RegisterBlock);
651 MaskForVGPRBlockOps[RegisterBlock] = Mask;
652 }
653
654 uint32_t getMaskForVGPRBlockOps(Register RegisterBlock) const {
655 return MaskForVGPRBlockOps[RegisterBlock];
656 }
657
658 bool hasMaskForVGPRBlockOps(Register RegisterBlock) const {
659 return MaskForVGPRBlockOps.inBounds(N: RegisterBlock);
660 }
661
662public:
663 SIMachineFunctionInfo(const SIMachineFunctionInfo &MFI) = default;
664 SIMachineFunctionInfo(const Function &F, const GCNSubtarget *STI);
665
666 MachineFunctionInfo *
667 clone(BumpPtrAllocator &Allocator, MachineFunction &DestMF,
668 const DenseMap<MachineBasicBlock *, MachineBasicBlock *> &Src2DstMBB)
669 const override;
670
671 bool initializeBaseYamlFields(const yaml::SIMachineFunctionInfo &YamlMFI,
672 const MachineFunction &MF,
673 PerFunctionMIParsingState &PFS,
674 SMDiagnostic &Error, SMRange &SourceRange);
675
676 void reserveWWMRegister(Register Reg) { WWMReservedRegs.insert(X: Reg); }
677 bool isWWMReg(Register Reg) const {
678 return Reg.isVirtual() ? checkFlag(Reg, Flag: AMDGPU::VirtRegFlag::WWM_REG)
679 : WWMReservedRegs.contains(key: Reg);
680 }
681
682 void updatePerLaneVGPRMask(BitVector &RegMask) { PerLaneVGPRMask = RegMask; }
683 BitVector getPerLaneVGPRMask() const { return PerLaneVGPRMask; }
684 void clearPerLaneVGPRAllocMask() { PerLaneVGPRMask.clear(); }
685
686 SIModeRegisterDefaults getMode() const { return Mode; }
687
688 ArrayRef<SIRegisterInfo::SpilledReg>
689 getSGPRSpillToVirtualVGPRLanes(int FrameIndex) const {
690 auto I = SGPRSpillsToVirtualVGPRLanes.find(Val: FrameIndex);
691 return (I == SGPRSpillsToVirtualVGPRLanes.end())
692 ? ArrayRef<SIRegisterInfo::SpilledReg>()
693 : ArrayRef(I->second);
694 }
695
696 ArrayRef<Register> getSGPRSpillVGPRs() const { return SpillVGPRs; }
697 ArrayRef<Register> getSGPRSpillPhysVGPRs() const { return SpillPhysVGPRs; }
698
699 const WWMSpillsMap &getWWMSpills() const { return WWMSpills; }
700 const ReservedRegSet &getWWMReservedRegs() const { return WWMReservedRegs; }
701
702 bool isWWMReservedRegister(Register Reg) const {
703 return WWMReservedRegs.contains(key: Reg);
704 }
705
706 bool isWholeWaveFunction() const { return IsWholeWaveFunction; }
707
708 ArrayRef<PrologEpilogSGPRSpill> getPrologEpilogSGPRSpills() const {
709 assert(is_sorted(PrologEpilogSGPRSpills, llvm::less_first()));
710 return PrologEpilogSGPRSpills;
711 }
712
713 GCNUserSGPRUsageInfo &getUserSGPRInfo() { return UserSGPRInfo; }
714
715 const GCNUserSGPRUsageInfo &getUserSGPRInfo() const { return UserSGPRInfo; }
716
717 void addToPrologEpilogSGPRSpills(Register Reg,
718 PrologEpilogSGPRSaveRestoreInfo SI) {
719 assert(!hasPrologEpilogSGPRSpillEntry(Reg));
720
721 // Insert a new entry in the right place to keep the vector in sorted order.
722 // This should be cheap since the vector is expected to be very short.
723 PrologEpilogSGPRSpills.insert(
724 I: upper_bound(
725 Range&: PrologEpilogSGPRSpills, Value&: Reg,
726 C: [](const auto &LHS, const auto &RHS) { return LHS < RHS.first; }),
727 Elt: std::make_pair(x&: Reg, y&: SI));
728 }
729
730 // Check if an entry created for \p Reg in PrologEpilogSGPRSpills. Return true
731 // on success and false otherwise.
732 bool hasPrologEpilogSGPRSpillEntry(Register Reg) const {
733 const auto *I = find_if(Range: PrologEpilogSGPRSpills, P: [&Reg](const auto &Spill) {
734 return Spill.first == Reg;
735 });
736 return I != PrologEpilogSGPRSpills.end();
737 }
738
739 // Get the scratch SGPR if allocated to save/restore \p Reg.
740 Register getScratchSGPRCopyDstReg(Register Reg) const {
741 const auto *I = find_if(Range: PrologEpilogSGPRSpills, P: [&Reg](const auto &Spill) {
742 return Spill.first == Reg;
743 });
744 if (I != PrologEpilogSGPRSpills.end() &&
745 I->second.getKind() == SGPRSaveKind::COPY_TO_SCRATCH_SGPR)
746 return I->second.getReg();
747
748 return AMDGPU::NoRegister;
749 }
750
751 // Get all scratch SGPRs allocated to copy/restore the SGPR spills.
752 void getAllScratchSGPRCopyDstRegs(SmallVectorImpl<Register> &Regs) const {
753 for (const auto &SI : PrologEpilogSGPRSpills) {
754 if (SI.second.getKind() == SGPRSaveKind::COPY_TO_SCRATCH_SGPR)
755 Regs.push_back(Elt: SI.second.getReg());
756 }
757 }
758
759 // Check if \p FI is allocated for any SGPR spill to a VGPR lane during PEI.
760 bool checkIndexInPrologEpilogSGPRSpills(int FI) const {
761 return find_if(Range: PrologEpilogSGPRSpills,
762 P: [FI](const std::pair<Register,
763 PrologEpilogSGPRSaveRestoreInfo> &SI) {
764 return SI.second.getKind() ==
765 SGPRSaveKind::SPILL_TO_VGPR_LANE &&
766 SI.second.getIndex() == FI;
767 }) != PrologEpilogSGPRSpills.end();
768 }
769
770 // Remove if an entry created for \p Reg.
771 void removePrologEpilogSGPRSpillEntry(Register Reg) {
772 auto I = find_if(Range&: PrologEpilogSGPRSpills,
773 P: [&Reg](const auto &Spill) { return Spill.first == Reg; });
774 if (I == PrologEpilogSGPRSpills.end())
775 return;
776
777 PrologEpilogSGPRSpills.erase(CI: I);
778 }
779
780 const PrologEpilogSGPRSaveRestoreInfo &
781 getPrologEpilogSGPRSaveRestoreInfo(Register Reg) const {
782 const auto *I = find_if(Range: PrologEpilogSGPRSpills, P: [&Reg](const auto &Spill) {
783 return Spill.first == Reg;
784 });
785 assert(I != PrologEpilogSGPRSpills.end());
786
787 return I->second;
788 }
789
790 ArrayRef<SIRegisterInfo::SpilledReg>
791 getSGPRSpillToPhysicalVGPRLanes(int FrameIndex) const {
792 auto I = SGPRSpillsToPhysicalVGPRLanes.find(Val: FrameIndex);
793 return (I == SGPRSpillsToPhysicalVGPRLanes.end())
794 ? ArrayRef<SIRegisterInfo::SpilledReg>()
795 : ArrayRef(I->second);
796 }
797
798 void setFlag(Register Reg, uint8_t Flag) {
799 assert(Reg.isVirtual());
800 if (VRegFlags.inBounds(N: Reg))
801 VRegFlags[Reg] |= Flag;
802 }
803
804 bool checkFlag(Register Reg, uint8_t Flag) const {
805 if (Reg.isPhysical())
806 return false;
807
808 return VRegFlags.inBounds(N: Reg) && VRegFlags[Reg] & Flag;
809 }
810
811 bool hasVRegFlags() { return VRegFlags.size(); }
812
813 void allocateWWMSpill(MachineFunction &MF, Register VGPR, uint64_t Size = 4,
814 Align Alignment = Align(4));
815
816 void splitWWMSpillRegisters(
817 MachineFunction &MF,
818 SmallVectorImpl<std::pair<Register, int>> &CalleeSavedRegs,
819 SmallVectorImpl<std::pair<Register, int>> &ScratchRegs) const;
820
821 ArrayRef<MCPhysReg> getAGPRSpillVGPRs() const {
822 return SpillAGPR;
823 }
824
825 Register getSGPRForEXECCopy() const { return SGPRForEXECCopy; }
826
827 void setSGPRForEXECCopy(Register Reg) { SGPRForEXECCopy = Reg; }
828
829 ArrayRef<MCPhysReg> getVGPRSpillAGPRs() const {
830 return SpillVGPR;
831 }
832
833 MCPhysReg getVGPRToAGPRSpill(int FrameIndex, unsigned Lane) const {
834 auto I = VGPRToAGPRSpills.find(Val: FrameIndex);
835 return (I == VGPRToAGPRSpills.end()) ? (MCPhysReg)AMDGPU::NoRegister
836 : I->second.Lanes[Lane];
837 }
838
839 void setVGPRToAGPRSpillDead(int FrameIndex) {
840 auto I = VGPRToAGPRSpills.find(Val: FrameIndex);
841 if (I != VGPRToAGPRSpills.end())
842 I->second.IsDead = true;
843 }
844
845 // To bring the allocated WWM registers in \p WWMVGPRs to the lowest available
846 // range.
847 void shiftWwmVGPRsToLowestRange(MachineFunction &MF,
848 SmallVectorImpl<Register> &WWMVGPRs,
849 BitVector &SavedVGPRs);
850
851 bool allocateSGPRSpillToVGPRLane(MachineFunction &MF, int FI,
852 bool SpillToPhysVGPRLane = false,
853 bool IsPrologEpilog = false);
854 bool allocateVGPRSpillToAGPR(MachineFunction &MF, int FI, bool isAGPRtoVGPR);
855
856 /// If \p ResetSGPRSpillStackIDs is true, reset the stack ID from sgpr-spill
857 /// to the default stack.
858 bool removeDeadFrameIndices(MachineFrameInfo &MFI,
859 bool ResetSGPRSpillStackIDs);
860
861 int getScavengeFI(MachineFrameInfo &MFI, const SIRegisterInfo &TRI);
862 std::optional<int> getOptionalScavengeFI() const { return ScavengeFI; }
863
864 void setNoWWMPoolSGPRSpillFallback() { HasNoWWMPoolSGPRSpillFallback = true; }
865 bool hasNoWWMPoolSGPRSpillFallback() const {
866 return HasNoWWMPoolSGPRSpillFallback;
867 }
868
869 unsigned getBytesInStackArgArea() const {
870 return BytesInStackArgArea;
871 }
872
873 void setBytesInStackArgArea(unsigned Bytes) {
874 BytesInStackArgArea = Bytes;
875 }
876
877 bool isDynamicVGPREnabled() const { return DynamicVGPRBlockSize != 0; }
878 unsigned getDynamicVGPRBlockSize() const { return DynamicVGPRBlockSize; }
879
880 // This is only used if we need to save any dynamic VGPRs in scratch.
881 unsigned getScratchReservedForDynamicVGPRs() const {
882 return ScratchReservedForDynamicVGPRs;
883 }
884
885 void setScratchReservedForDynamicVGPRs(unsigned SizeInBytes) {
886 ScratchReservedForDynamicVGPRs = SizeInBytes;
887 }
888
889 // Add user SGPRs.
890 Register addPrivateSegmentBuffer(const SIRegisterInfo &TRI);
891 Register addDispatchPtr(const SIRegisterInfo &TRI);
892 Register addQueuePtr(const SIRegisterInfo &TRI);
893 Register addKernargSegmentPtr(const SIRegisterInfo &TRI);
894 Register addDispatchID(const SIRegisterInfo &TRI);
895 Register addFlatScratchInit(const SIRegisterInfo &TRI);
896 Register addPrivateSegmentSize(const SIRegisterInfo &TRI);
897 Register addImplicitBufferPtr(const SIRegisterInfo &TRI);
898 Register addLDSKernelId();
899 SmallVectorImpl<MCRegister> *
900 addPreloadedKernArg(const SIRegisterInfo &TRI, const TargetRegisterClass *RC,
901 unsigned AllocSizeDWord, int KernArgIdx,
902 int PaddingSGPRs);
903
904 /// Increment user SGPRs used for padding the argument list only.
905 Register addReservedUserSGPR() {
906 Register Next = getNextUserSGPR();
907 ++NumUserSGPRs;
908 return Next;
909 }
910
911 // Add system SGPRs.
912 Register addWorkGroupIDX() {
913 ArgInfo.WorkGroupIDX = ArgDescriptor::createRegister(Reg: getNextSystemSGPR());
914 NumSystemSGPRs += 1;
915 return ArgInfo.WorkGroupIDX.getRegister();
916 }
917
918 Register addWorkGroupIDY() {
919 ArgInfo.WorkGroupIDY = ArgDescriptor::createRegister(Reg: getNextSystemSGPR());
920 NumSystemSGPRs += 1;
921 return ArgInfo.WorkGroupIDY.getRegister();
922 }
923
924 Register addWorkGroupIDZ() {
925 ArgInfo.WorkGroupIDZ = ArgDescriptor::createRegister(Reg: getNextSystemSGPR());
926 NumSystemSGPRs += 1;
927 return ArgInfo.WorkGroupIDZ.getRegister();
928 }
929
930 Register addWorkGroupInfo() {
931 ArgInfo.WorkGroupInfo = ArgDescriptor::createRegister(Reg: getNextSystemSGPR());
932 NumSystemSGPRs += 1;
933 return ArgInfo.WorkGroupInfo.getRegister();
934 }
935
936 bool hasLDSKernelId() const { return LDSKernelId; }
937
938 // Add special VGPR inputs
939 void setWorkItemIDX(ArgDescriptor Arg) {
940 ArgInfo.WorkItemIDX = Arg;
941 }
942
943 void setWorkItemIDY(ArgDescriptor Arg) {
944 ArgInfo.WorkItemIDY = Arg;
945 }
946
947 void setWorkItemIDZ(ArgDescriptor Arg) {
948 ArgInfo.WorkItemIDZ = Arg;
949 }
950
951 Register addPrivateSegmentWaveByteOffset() {
952 ArgInfo.PrivateSegmentWaveByteOffset
953 = ArgDescriptor::createRegister(Reg: getNextSystemSGPR());
954 NumSystemSGPRs += 1;
955 return ArgInfo.PrivateSegmentWaveByteOffset.getRegister();
956 }
957
958 void setPrivateSegmentWaveByteOffset(Register Reg) {
959 ArgInfo.PrivateSegmentWaveByteOffset = ArgDescriptor::createRegister(Reg);
960 }
961
962 bool hasWorkGroupIDX() const {
963 return WorkGroupIDX;
964 }
965
966 bool hasWorkGroupIDY() const {
967 return WorkGroupIDY;
968 }
969
970 bool hasWorkGroupIDZ() const {
971 return WorkGroupIDZ;
972 }
973
974 bool hasWorkGroupInfo() const {
975 return WorkGroupInfo;
976 }
977
978 bool hasPrivateSegmentWaveByteOffset() const {
979 return PrivateSegmentWaveByteOffset;
980 }
981
982 bool hasWorkItemIDX() const {
983 return WorkItemIDX;
984 }
985
986 bool hasWorkItemIDY() const {
987 return WorkItemIDY;
988 }
989
990 bool hasWorkItemIDZ() const {
991 return WorkItemIDZ;
992 }
993
994 bool hasImplicitArgPtr() const {
995 return ImplicitArgPtr;
996 }
997
998 AMDGPUFunctionArgInfo &getArgInfo() {
999 return ArgInfo;
1000 }
1001
1002 const AMDGPUFunctionArgInfo &getArgInfo() const {
1003 return ArgInfo;
1004 }
1005
1006 std::tuple<const ArgDescriptor *, const TargetRegisterClass *, LLT>
1007 getPreloadedValue(AMDGPUFunctionArgInfo::PreloadedValue Value) const {
1008 return ArgInfo.getPreloadedValue(Value);
1009 }
1010
1011 MCRegister getPreloadedReg(AMDGPUFunctionArgInfo::PreloadedValue Value) const {
1012 const auto *Arg = std::get<0>(t: ArgInfo.getPreloadedValue(Value));
1013 return Arg ? Arg->getRegister() : MCRegister();
1014 }
1015
1016 unsigned getGITPtrHigh() const {
1017 return GITPtrHigh;
1018 }
1019
1020 Register getGITPtrLoReg(const MachineFunction &MF) const;
1021
1022 uint32_t get32BitAddressHighBits() const {
1023 return HighBitsOf32BitAddress;
1024 }
1025
1026 unsigned getNumUserSGPRs() const {
1027 return NumUserSGPRs;
1028 }
1029
1030 unsigned getNumPreloadedSGPRs() const {
1031 return NumUserSGPRs + NumSystemSGPRs;
1032 }
1033
1034 unsigned getNumKernargPreloadedSGPRs() const {
1035 return UserSGPRInfo.getNumKernargPreloadSGPRs();
1036 }
1037
1038 unsigned getNumWaveDispatchSGPRs() const { return NumWaveDispatchSGPRs; }
1039
1040 void setNumWaveDispatchSGPRs(unsigned Count) { NumWaveDispatchSGPRs = Count; }
1041
1042 unsigned getNumWaveDispatchVGPRs() const { return NumWaveDispatchVGPRs; }
1043
1044 void setNumWaveDispatchVGPRs(unsigned Count) { NumWaveDispatchVGPRs = Count; }
1045
1046 Register getPrivateSegmentWaveByteOffsetSystemSGPR() const {
1047 if (ArgInfo.PrivateSegmentWaveByteOffset)
1048 return ArgInfo.PrivateSegmentWaveByteOffset.getRegister();
1049 return MCRegister();
1050 }
1051
1052 /// Returns the physical register reserved for use as the resource
1053 /// descriptor for scratch accesses.
1054 Register getScratchRSrcReg() const {
1055 return ScratchRSrcReg;
1056 }
1057
1058 void setScratchRSrcReg(Register Reg) {
1059 assert(Reg != 0 && "Should never be unset");
1060 ScratchRSrcReg = Reg;
1061 }
1062
1063 Register getFrameOffsetReg() const {
1064 return FrameOffsetReg;
1065 }
1066
1067 void setFrameOffsetReg(Register Reg) {
1068 assert(Reg != 0 && "Should never be unset");
1069 FrameOffsetReg = Reg;
1070 }
1071
1072 void setStackPtrOffsetReg(Register Reg) {
1073 assert(Reg != 0 && "Should never be unset");
1074 StackPtrOffsetReg = Reg;
1075 }
1076
1077 void setLongBranchReservedReg(Register Reg) { LongBranchReservedReg = Reg; }
1078
1079 // Note the unset value for this is AMDGPU::SP_REG rather than
1080 // NoRegister. This is mostly a workaround for MIR tests where state that
1081 // can't be directly computed from the function is not preserved in serialized
1082 // MIR.
1083 Register getStackPtrOffsetReg() const {
1084 return StackPtrOffsetReg;
1085 }
1086
1087 Register getLongBranchReservedReg() const { return LongBranchReservedReg; }
1088
1089 Register getQueuePtrUserSGPR() const {
1090 return ArgInfo.QueuePtr.getRegister();
1091 }
1092
1093 Register getImplicitBufferPtrUserSGPR() const {
1094 return ArgInfo.ImplicitBufferPtr.getRegister();
1095 }
1096
1097 bool hasSpilledSGPRs() const {
1098 return HasSpilledSGPRs;
1099 }
1100
1101 void setHasSpilledSGPRs(bool Spill = true) {
1102 HasSpilledSGPRs = Spill;
1103 }
1104
1105 bool hasSpilledVGPRs() const {
1106 return HasSpilledVGPRs;
1107 }
1108
1109 void setHasSpilledVGPRs(bool Spill = true) {
1110 HasSpilledVGPRs = Spill;
1111 }
1112
1113 bool hasNonSpillStackObjects() const {
1114 return HasNonSpillStackObjects;
1115 }
1116
1117 void setHasNonSpillStackObjects(bool StackObject = true) {
1118 HasNonSpillStackObjects = StackObject;
1119 }
1120
1121 bool isStackRealigned() const {
1122 return IsStackRealigned;
1123 }
1124
1125 void setIsStackRealigned(bool Realigned = true) {
1126 IsStackRealigned = Realigned;
1127 }
1128
1129 unsigned getNumSpilledSGPRs() const {
1130 return NumSpilledSGPRs;
1131 }
1132
1133 unsigned getNumSpilledVGPRs() const {
1134 return NumSpilledVGPRs;
1135 }
1136
1137 void addToSpilledSGPRs(unsigned num) {
1138 NumSpilledSGPRs += num;
1139 }
1140
1141 void addToSpilledVGPRs(unsigned num) {
1142 NumSpilledVGPRs += num;
1143 }
1144
1145 unsigned getPSInputAddr() const {
1146 return PSInputAddr;
1147 }
1148
1149 unsigned getPSInputEnable() const {
1150 return PSInputEnable;
1151 }
1152
1153 bool isPSInputAllocated(unsigned Index) const {
1154 return PSInputAddr & (1 << Index);
1155 }
1156
1157 void markPSInputAllocated(unsigned Index) {
1158 PSInputAddr |= 1 << Index;
1159 }
1160
1161 void markPSInputEnabled(unsigned Index) {
1162 PSInputEnable |= 1 << Index;
1163 }
1164
1165 bool returnsVoid() const {
1166 return ReturnsVoid;
1167 }
1168
1169 void setIfReturnsVoid(bool Value) {
1170 ReturnsVoid = Value;
1171 }
1172
1173 /// \returns A pair of default/requested minimum/maximum flat work group sizes
1174 /// for this function.
1175 std::pair<unsigned, unsigned> getFlatWorkGroupSizes() const {
1176 return FlatWorkGroupSizes;
1177 }
1178
1179 /// \returns Default/requested minimum flat work group size for this function.
1180 unsigned getMinFlatWorkGroupSize() const {
1181 return FlatWorkGroupSizes.first;
1182 }
1183
1184 /// \returns Default/requested maximum flat work group size for this function.
1185 unsigned getMaxFlatWorkGroupSize() const {
1186 return FlatWorkGroupSizes.second;
1187 }
1188
1189 /// \returns A pair of default/requested minimum/maximum number of waves per
1190 /// execution unit.
1191 std::pair<unsigned, unsigned> getWavesPerEU() const {
1192 return WavesPerEU;
1193 }
1194
1195 /// \returns Default/requested minimum number of waves per execution unit.
1196 unsigned getMinWavesPerEU() const {
1197 return WavesPerEU.first;
1198 }
1199
1200 /// \returns Default/requested maximum number of waves per execution unit.
1201 unsigned getMaxWavesPerEU() const {
1202 return WavesPerEU.second;
1203 }
1204
1205 const AMDGPUGWSResourcePseudoSourceValue *
1206 getGWSPSV(const AMDGPUTargetMachine &TM) {
1207 return &GWSResourcePSV;
1208 }
1209
1210 unsigned getOccupancy() const {
1211 return Occupancy;
1212 }
1213
1214 unsigned getMinAllowedOccupancy() const {
1215 if (!isMemoryBound() && !needsWaveLimiter())
1216 return Occupancy;
1217 return (Occupancy < 4) ? Occupancy : 4;
1218 }
1219
1220 void limitOccupancy(const MachineFunction &MF);
1221
1222 void limitOccupancy(unsigned Limit) {
1223 if (Occupancy > Limit)
1224 Occupancy = Limit;
1225 }
1226
1227 void increaseOccupancy(const MachineFunction &MF, unsigned Limit) {
1228 if (Occupancy < Limit)
1229 Occupancy = Limit;
1230 limitOccupancy(MF);
1231 }
1232
1233 unsigned getMaxMemoryClusterDWords() const { return MaxMemoryClusterDWords; }
1234
1235 unsigned getMinNumAGPRs() const { return MinNumAGPRs; }
1236
1237 /// Return true if an MFMA that requires at least \p NumRegs should select to
1238 /// the AGPR form, instead of the VGPR form.
1239 bool selectAGPRFormMFMA(unsigned NumRegs) const {
1240 return !MFMAVGPRForm && getMinNumAGPRs() >= NumRegs;
1241 }
1242
1243 // \returns true if a function has a use of AGPRs via inline asm or
1244 // has a call which may use it.
1245 bool mayUseAGPRs(const Function &F) const;
1246
1247 /// \returns Default/requested number of work groups for this function.
1248 SmallVector<unsigned> getMaxNumWorkGroups() const { return MaxNumWorkGroups; }
1249
1250 unsigned getMaxNumWorkGroupsX() const { return MaxNumWorkGroups[0]; }
1251 unsigned getMaxNumWorkGroupsY() const { return MaxNumWorkGroups[1]; }
1252 unsigned getMaxNumWorkGroupsZ() const { return MaxNumWorkGroups[2]; }
1253
1254 AMDGPU::ClusterDimsAttr getClusterDims() const { return ClusterDims; }
1255};
1256
1257} // end namespace llvm
1258
1259#endif // LLVM_LIB_TARGET_AMDGPU_SIMACHINEFUNCTIONINFO_H
1260