1//===- AMDGPURegBankLegalizeRules --------------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUREGBANKLEGALIZERULES_H
10#define LLVM_LIB_TARGET_AMDGPU_AMDGPUREGBANKLEGALIZERULES_H
11
12#include "llvm/ADT/DenseMap.h"
13#include "llvm/ADT/SmallVector.h"
14#include <functional>
15
16namespace llvm {
17
18class LLT;
19class MachineRegisterInfo;
20class MachineInstr;
21class GCNSubtarget;
22class MachineFunction;
23template <typename T> class GenericUniformityInfo;
24template <typename T> class GenericSSAContext;
25using MachineSSAContext = GenericSSAContext<MachineFunction>;
26using MachineUniformityInfo = GenericUniformityInfo<MachineSSAContext>;
27
28namespace AMDGPU {
29
30/// \returns true if \p Ty is a pointer type with size \p Width.
31bool isAnyPtr(LLT Ty, unsigned Width);
32
33// IDs used to build predicate for RegBankLegalizeRule. Predicate can have one
34// or more IDs and each represents a check for 'uniform or divergent' + LLT or
35// just LLT on register operand.
36// Most often checking one operand is enough to decide which RegBankLLTMapping
37// to apply (see Fast Rules), IDs are useful when two or more operands need to
38// be checked.
39enum UniformityLLTOpPredicateID {
40 // Represents non-register and physical register operands.
41 _,
42 // scalars
43 S1,
44 S16,
45 S32,
46 S64,
47 S128,
48
49 UniS1,
50 UniS16,
51 UniS32,
52 UniS64,
53 UniS128,
54
55 UniBF16,
56
57 DivS1,
58 DivS16,
59 DivS32,
60 DivS64,
61 DivS128,
62
63 // any LLT, divergent-check only predicate
64 DivAnyTy,
65
66 // pointers
67 P0,
68 P1,
69 P2,
70 P3,
71 P4,
72 P5,
73 P8,
74 Ptr32,
75 Ptr64,
76 Ptr128,
77
78 UniP0,
79 UniP1,
80 UniP2,
81 UniP3,
82 UniP4,
83 UniP5,
84 UniP6,
85 UniP8,
86 UniPtr32,
87 UniPtr64,
88 UniPtr128,
89
90 DivP0,
91 DivP1,
92 DivP2,
93 DivP3,
94 DivP4,
95 DivP5,
96 DivPtr32,
97 DivPtr64,
98 DivPtr128,
99
100 // vectors
101 V2S16,
102 V2S32,
103 V2S64,
104 V3S32,
105 V4S32,
106 V32S32,
107
108 UniV2S16,
109 UniV2S32,
110 UniV4S32,
111 UniV2S64,
112 UniV3S32,
113 UniV6S32,
114 UniV8S16,
115 UniV8S32,
116 UniV16S16,
117 UniV16S32,
118 UniV32S16,
119 UniV32S32,
120
121 DivV2S16,
122 DivV2S32,
123 DivV4S32,
124 DivV2S64,
125 DivV3S32,
126 DivV4S16,
127 DivV8S16,
128 DivV8S32,
129 DivV16S16,
130 DivV16S32,
131 DivV6S32,
132 DivV32S16,
133 DivV32S32,
134
135 // B types
136 B32,
137 B64,
138 B96,
139 B128,
140 B160,
141 B256,
142 B512,
143 BRC,
144
145 UniB32,
146 UniB64,
147 UniB96,
148 UniB128,
149 UniB160,
150 UniB256,
151 UniB512,
152 UniBRC,
153
154 DivB32,
155 DivB64,
156 DivB96,
157 DivB128,
158 DivB160,
159 DivB256,
160 DivB512,
161 DivBRC
162};
163
164// How to apply register bank on register operand.
165// In most cases, this serves as a LLT and register bank assert.
166// Can change operands and insert copies, extends, truncs, and read-any-lanes.
167// Anything more complicated requires LoweringMethod.
168enum RegBankLLTMappingApplyID {
169 InvalidMapping,
170 None,
171 IntrId,
172 Imm,
173 Vcc,
174
175 // any LLT, bank-only apply IDs
176 VgprAnyTy,
177 AgprAnyTy,
178 VgprOrAgprAnyTy,
179
180 // sgpr scalars, pointers, vectors and B-types
181 Sgpr16,
182 Sgpr32,
183 Sgpr64,
184 Sgpr128,
185 SgprP0,
186 SgprP1,
187 SgprP2,
188 SgprP3,
189 SgprP4,
190 SgprP5,
191 SgprP6,
192 SgprP8,
193 SgprPtr32,
194 SgprPtr64,
195 SgprPtr128,
196 SgprV2S16,
197 SgprV4S32,
198 SgprV2S32,
199 SgprB32,
200 SgprB64,
201 SgprB96,
202 SgprB128,
203 SgprB256,
204 SgprB512,
205 SgprBRC,
206
207 // vgpr scalars, pointers, vectors and B-types
208 Vgpr16,
209 Vgpr32,
210 Vgpr64,
211 Vgpr128,
212 VgprP0,
213 VgprP1,
214 VgprP2,
215 VgprP3,
216 VgprP4,
217 VgprP5,
218 VgprPtr32,
219 VgprPtr64,
220 VgprPtr128,
221 VgprV2S16,
222 VgprV2S32,
223 VgprV3S32,
224 VgprB32,
225 VgprB64,
226 VgprB96,
227 VgprB128,
228 VgprB160,
229 VgprB256,
230 VgprB512,
231 VgprBRC,
232 VgprV4S16,
233 VgprV8S16,
234 VgprV16S16,
235 VgprV4S32,
236 VgprV8S32,
237 VgprV2S64,
238
239 // Dst only modifiers: read-any-lane and truncs
240 UniInVcc,
241 UniInVgprS16,
242 UniInVgprS32,
243 UniInVgprS64,
244 UniInVgprV2S16,
245 UniInVgprV2S32,
246 UniInVgprV3S32,
247 UniInVgprV4S32,
248 UniInVgprV2S64,
249 UniInVgprV6S32,
250 UniInVgprV8S16,
251 UniInVgprV8S32,
252 UniInVgprV16S16,
253 UniInVgprV16S32,
254 UniInVgprV32S16,
255 UniInVgprV32S32,
256 UniInVgprB32,
257 UniInVgprB64,
258 UniInVgprB96,
259 UniInVgprB128,
260 UniInVgprB160,
261 UniInVgprB256,
262 UniInVgprB512,
263
264 Sgpr32Trunc,
265
266 // Dst only modifiers: dst was assigned VGPR by RegBankSelect but the
267 // instruction result must be in SGPR. Replace dst with SGPR, then copy the
268 // result back to the original VGPR.
269 Sgpr32ToVgprDst,
270 Sgpr64ToVgprDst,
271
272 // Src only modifiers: execute in waterfall loop if divergent
273 Sgpr32_WF,
274 SgprV4S32_WF,
275
276 // Src only modifiers: execute in waterfall loop for calls
277 SgprP0Call_WF,
278 SgprP4Call_WF,
279
280 // Src only modifiers: for operands that must end up in M0. If divergent,
281 // readfirstlane to SGPR. The result can then be copied to M0 in ISel.
282 SgprB32_M0,
283
284 // Src only modifiers: operand must be SGPR, if in VGPR, insert readfirstlane
285 // to move to SGPR.
286 SgprB32_ReadFirstLane,
287 SgprB64_ReadFirstLane,
288 SgprV4S32_ReadFirstLane,
289 SgprV8S32_ReadFirstLane,
290
291 // Src only modifiers: extends
292 Sgpr32AExt,
293 Sgpr32AExtBoolInReg,
294 Sgpr32SExt,
295 Sgpr32ZExt,
296 Vgpr32AExt,
297 Vgpr32SExt,
298 Vgpr32ZExt,
299
300 VgprV6S32,
301 VgprV16S32,
302 VgprV32S16,
303 VgprV32S32,
304};
305
306// Instruction needs to be replaced with sequence of instructions. Lowering was
307// not done by legalizer since instructions is available in either sgpr or vgpr.
308// For example S64 AND is available on sgpr, for that reason S64 AND is legal in
309// context of Legalizer that only checks LLT. But S64 AND is not available on
310// vgpr. Lower it to two S32 vgpr ANDs.
311enum LoweringMethodID {
312 DoNotLower,
313 VccExtToSel,
314 UniExtToSel,
315 UnpackBitShift,
316 UnpackMinMax,
317 S_BFE,
318 V_BFE,
319 VgprToVccCopy,
320 UniMAD64,
321 UniMul64,
322 DivSMulToMAD,
323 SplitTo32,
324 SplitTo32Mul,
325 ScalarizeToS16,
326 SplitTo32Select,
327 SplitTo32SExtInReg,
328 S_BUF_to_BUF,
329 Ext32To64,
330 UniCstExt,
331 CtPop64To32,
332 SplitLoad,
333 WidenLoad,
334 WidenMMOToS32,
335 UnpackAExt,
336 VerifyAllSgpr,
337 ApplyAllVgpr,
338 UnmergeToShiftTrunc,
339 AextToS32InIncomingBlockGPHI,
340 VerifyAllSgprGPHI,
341 VerifyAllSgprOrVgprGPHI,
342 ApplyINTRIN_IMAGE,
343 ApplyBVH_INTERSECT_RAY,
344 SplitBitCount64To32,
345 ExtrVecEltToSel,
346 ExtrVecEltTo32,
347 InsVecEltToSel,
348 InsVecEltTo32,
349 AbsToNegMax,
350 AbsToS32,
351 DynStackAlloc,
352 DeletePrefetch,
353 LowerSetRounding,
354 LowerGetRounding
355};
356
357enum FastRulesTypes {
358 NoFastRules,
359 Standard, // S16, S32, S64, V2S16
360 StandardB, // B32, B64, B96, B128
361 Vector, // S32, V2S32, V3S32, V4S32
362};
363
364struct RegBankLLTMapping {
365 SmallVector<RegBankLLTMappingApplyID, 2> DstOpMapping;
366 SmallVector<RegBankLLTMappingApplyID, 4> SrcOpMapping;
367 LoweringMethodID LoweringMethod;
368 RegBankLLTMapping(
369 std::initializer_list<RegBankLLTMappingApplyID> DstOpMappingList,
370 std::initializer_list<RegBankLLTMappingApplyID> SrcOpMappingList,
371 LoweringMethodID LoweringMethod = DoNotLower);
372};
373
374struct PredicateMapping {
375 SmallVector<UniformityLLTOpPredicateID, 4> OpUniformityAndTypes;
376 std::function<bool(const MachineInstr &)> TestFunc;
377 PredicateMapping(
378 std::initializer_list<UniformityLLTOpPredicateID> OpList,
379 std::function<bool(const MachineInstr &)> TestFunc = nullptr);
380
381 bool match(const MachineInstr &MI, const MachineUniformityInfo &MUI,
382 const MachineRegisterInfo &MRI) const;
383};
384
385struct RegBankLegalizeRule {
386 PredicateMapping Predicate;
387 RegBankLLTMapping OperandMapping;
388};
389
390class SetOfRulesForOpcode {
391 // "Slow Rules". More complex 'Rules[i].Predicate', check them one by one.
392 SmallVector<RegBankLegalizeRule, 4> Rules;
393
394 // "Fast Rules"
395 // Instead of testing each 'Rules[i].Predicate' we do direct access to
396 // RegBankLLTMapping using getFastPredicateSlot. For example if:
397 // - FastTypes == Standard Uni[0] holds Mapping in case Op 0 is uniform S32
398 // - FastTypes == Vector Div[3] holds Mapping in case Op 0 is divergent V4S32
399 FastRulesTypes FastTypes = NoFastRules;
400#define InvMapping RegBankLLTMapping({InvalidMapping}, {InvalidMapping})
401 RegBankLLTMapping Uni[4] = {InvMapping, InvMapping, InvMapping, InvMapping};
402 RegBankLLTMapping Div[4] = {InvMapping, InvMapping, InvMapping, InvMapping};
403
404public:
405 SetOfRulesForOpcode();
406 SetOfRulesForOpcode(FastRulesTypes FastTypes);
407
408 const RegBankLLTMapping *
409 findMappingForMI(const MachineInstr &MI, const MachineRegisterInfo &MRI,
410 const MachineUniformityInfo &MUI) const;
411
412 void addRule(RegBankLegalizeRule Rule);
413
414 void addFastRuleDivergent(UniformityLLTOpPredicateID Ty,
415 RegBankLLTMapping RuleApplyIDs);
416 void addFastRuleUniform(UniformityLLTOpPredicateID Ty,
417 RegBankLLTMapping RuleApplyIDs);
418
419private:
420 int getFastPredicateSlot(UniformityLLTOpPredicateID Ty) const;
421};
422
423// Essentially 'map<Opcode(or intrinsic_opcode), SetOfRulesForOpcode>' but a
424// little more efficient.
425class RegBankLegalizeRules {
426 const GCNSubtarget *ST;
427 MachineRegisterInfo *MRI;
428 // Separate maps for G-opcodes and intrinsics since they are in different
429 // enums. Multiple opcodes can share same set of rules.
430 // RulesAlias = map<Opcode, KeyOpcode>
431 // Rules = map<KeyOpcode, SetOfRulesForOpcode>
432 SmallDenseMap<unsigned, unsigned, 256> GRulesAlias;
433 SmallDenseMap<unsigned, SetOfRulesForOpcode, 128> GRules;
434 SmallDenseMap<unsigned, unsigned, 128> IRulesAlias;
435 SmallDenseMap<unsigned, SetOfRulesForOpcode, 64> IRules;
436 class RuleSetInitializer {
437 SetOfRulesForOpcode *RuleSet;
438
439 public:
440 // Used for clang-format line breaks and to force writing all rules for
441 // opcode in same place.
442 template <class AliasMap, class RulesMap>
443 RuleSetInitializer(std::initializer_list<unsigned> OpcList,
444 AliasMap &RulesAlias, RulesMap &Rules,
445 FastRulesTypes FastTypes = NoFastRules) {
446 unsigned KeyOpcode = *OpcList.begin();
447 for (unsigned Opc : OpcList) {
448 [[maybe_unused]] auto [_, NewInput] =
449 RulesAlias.try_emplace(Opc, KeyOpcode);
450 assert(NewInput && "Can't redefine existing Rules");
451 }
452
453 auto [DenseMapIter, NewInput] = Rules.try_emplace(KeyOpcode, FastTypes);
454 assert(NewInput && "Can't redefine existing Rules");
455
456 RuleSet = &DenseMapIter->second;
457 }
458
459 RuleSetInitializer(const RuleSetInitializer &) = delete;
460 RuleSetInitializer &operator=(const RuleSetInitializer &) = delete;
461 RuleSetInitializer(RuleSetInitializer &&) = delete;
462 RuleSetInitializer &operator=(RuleSetInitializer &&) = delete;
463 ~RuleSetInitializer() = default;
464
465 RuleSetInitializer &Div(UniformityLLTOpPredicateID Ty,
466 RegBankLLTMapping RuleApplyIDs,
467 bool STPred = true) {
468 if (STPred)
469 RuleSet->addFastRuleDivergent(Ty, RuleApplyIDs);
470 return *this;
471 }
472
473 RuleSetInitializer &Uni(UniformityLLTOpPredicateID Ty,
474 RegBankLLTMapping RuleApplyIDs,
475 bool STPred = true) {
476 if (STPred)
477 RuleSet->addFastRuleUniform(Ty, RuleApplyIDs);
478 return *this;
479 }
480
481 RuleSetInitializer &Any(RegBankLegalizeRule Init, bool STPred = true) {
482 if (STPred)
483 RuleSet->addRule(Rule: Init);
484 return *this;
485 }
486 };
487
488 RuleSetInitializer addRulesForGOpcs(std::initializer_list<unsigned> OpcList,
489 FastRulesTypes FastTypes = NoFastRules);
490
491 RuleSetInitializer addRulesForIOpcs(std::initializer_list<unsigned> OpcList,
492 FastRulesTypes FastTypes = NoFastRules);
493
494public:
495 // Initialize rules for all opcodes.
496 RegBankLegalizeRules(const GCNSubtarget &ST, MachineRegisterInfo &MRI);
497
498 // In case we don't want to regenerate same rules, we can use already
499 // generated rules but need to refresh references to objects that are
500 // created for this run.
501 void refreshRefs(const GCNSubtarget &_ST, MachineRegisterInfo &_MRI) {
502 ST = &_ST;
503 MRI = &_MRI;
504 };
505
506 const SetOfRulesForOpcode *getRulesForOpc(MachineInstr &MI) const;
507};
508
509} // end namespace AMDGPU
510} // end namespace llvm
511
512#endif
513