1//===-- AMDGPURegBankLegalizeRules.cpp ------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// Definitions of RegBankLegalize Rules for all opcodes.
10/// Implementation of container for all the Rules and search.
11/// Fast search for most common case when Rule.Predicate checks LLT and
12/// uniformity of register in operand 0.
13//
14//===----------------------------------------------------------------------===//
15
16#include "AMDGPURegBankLegalizeRules.h"
17#include "AMDGPUInstrInfo.h"
18#include "GCNSubtarget.h"
19#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
20#include "llvm/CodeGen/MachineUniformityAnalysis.h"
21#include "llvm/IR/IntrinsicsAMDGPU.h"
22#include "llvm/Support/AMDGPUAddrSpace.h"
23
24#define DEBUG_TYPE "amdgpu-regbanklegalize"
25
26using namespace llvm;
27using namespace AMDGPU;
28
29bool AMDGPU::isAnyPtr(LLT Ty, unsigned Width) {
30 return Ty.isPointer() && Ty.getSizeInBits() == Width;
31}
32
33RegBankLLTMapping::RegBankLLTMapping(
34 std::initializer_list<RegBankLLTMappingApplyID> DstOpMappingList,
35 std::initializer_list<RegBankLLTMappingApplyID> SrcOpMappingList,
36 LoweringMethodID LoweringMethod)
37 : DstOpMapping(DstOpMappingList), SrcOpMapping(SrcOpMappingList),
38 LoweringMethod(LoweringMethod) {}
39
40PredicateMapping::PredicateMapping(
41 std::initializer_list<UniformityLLTOpPredicateID> OpList,
42 std::function<bool(const MachineInstr &)> TestFunc)
43 : OpUniformityAndTypes(OpList), TestFunc(TestFunc) {}
44
45bool matchUniformityAndLLT(Register Reg, UniformityLLTOpPredicateID UniID,
46 const MachineUniformityInfo &MUI,
47 const MachineRegisterInfo &MRI) {
48 switch (UniID) {
49 case S1:
50 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 1);
51 case S16:
52 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 16);
53 case S32:
54 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 32);
55 case S64:
56 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 64);
57 case S128:
58 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 128);
59 case P0:
60 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 0, SizeInBits: 64);
61 case P1:
62 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 1, SizeInBits: 64);
63 case P2:
64 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 2, SizeInBits: 32);
65 case P3:
66 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 3, SizeInBits: 32);
67 case P4:
68 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 4, SizeInBits: 64);
69 case P5:
70 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 5, SizeInBits: 32);
71 case P8:
72 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 8, SizeInBits: 128);
73 case Ptr32:
74 return isAnyPtr(Ty: MRI.getType(Reg), Width: 32);
75 case Ptr64:
76 return isAnyPtr(Ty: MRI.getType(Reg), Width: 64);
77 case Ptr128:
78 return isAnyPtr(Ty: MRI.getType(Reg), Width: 128);
79 case V2S16:
80 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16);
81 case V2S32:
82 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32);
83 case V3S32:
84 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 32);
85 case V4S32:
86 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32);
87 case B32:
88 return MRI.getType(Reg).getSizeInBits() == 32;
89 case B64:
90 return MRI.getType(Reg).getSizeInBits() == 64;
91 case B96:
92 return MRI.getType(Reg).getSizeInBits() == 96;
93 case B128:
94 return MRI.getType(Reg).getSizeInBits() == 128;
95 case B160:
96 return MRI.getType(Reg).getSizeInBits() == 160;
97 case B256:
98 return MRI.getType(Reg).getSizeInBits() == 256;
99 case B512:
100 return MRI.getType(Reg).getSizeInBits() == 512;
101 case DivAnyTy:
102 return MUI.isDivergentAtDef(V: Reg);
103 case UniS1:
104 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 1) && MUI.isUniformAtDef(V: Reg);
105 case UniS16:
106 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 16) && MUI.isUniformAtDef(V: Reg);
107 case UniS32:
108 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 32) && MUI.isUniformAtDef(V: Reg);
109 case UniS64:
110 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 64) && MUI.isUniformAtDef(V: Reg);
111 case UniS128:
112 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 128) && MUI.isUniformAtDef(V: Reg);
113 case UniP0:
114 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 0, SizeInBits: 64) && MUI.isUniformAtDef(V: Reg);
115 case UniP1:
116 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 1, SizeInBits: 64) && MUI.isUniformAtDef(V: Reg);
117 case UniP2:
118 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 2, SizeInBits: 32) && MUI.isUniformAtDef(V: Reg);
119 case UniP3:
120 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 3, SizeInBits: 32) && MUI.isUniformAtDef(V: Reg);
121 case UniP4:
122 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 4, SizeInBits: 64) && MUI.isUniformAtDef(V: Reg);
123 case UniP5:
124 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 5, SizeInBits: 32) && MUI.isUniformAtDef(V: Reg);
125 case UniP6:
126 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 6, SizeInBits: 32) && MUI.isUniformAtDef(V: Reg);
127 case UniP8:
128 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 8, SizeInBits: 128) && MUI.isUniformAtDef(V: Reg);
129 case UniPtr32:
130 return isAnyPtr(Ty: MRI.getType(Reg), Width: 32) && MUI.isUniformAtDef(V: Reg);
131 case UniPtr64:
132 return isAnyPtr(Ty: MRI.getType(Reg), Width: 64) && MUI.isUniformAtDef(V: Reg);
133 case UniPtr128:
134 return isAnyPtr(Ty: MRI.getType(Reg), Width: 128) && MUI.isUniformAtDef(V: Reg);
135 case UniV2S16:
136 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16) &&
137 MUI.isUniformAtDef(V: Reg);
138 case UniV2S32:
139 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32) &&
140 MUI.isUniformAtDef(V: Reg);
141 case UniV3S32:
142 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 32) &&
143 MUI.isUniformAtDef(V: Reg);
144 case UniV4S32:
145 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32) &&
146 MUI.isUniformAtDef(V: Reg);
147 case UniV6S32:
148 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 6, ScalarSizeInBits: 32) &&
149 MUI.isUniformAtDef(V: Reg);
150 case UniV8S16:
151 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16) &&
152 MUI.isUniformAtDef(V: Reg);
153 case UniV8S32:
154 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 32) &&
155 MUI.isUniformAtDef(V: Reg);
156 case UniV16S16:
157 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 16) &&
158 MUI.isUniformAtDef(V: Reg);
159 case UniV16S32:
160 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 32) &&
161 MUI.isUniformAtDef(V: Reg);
162 case UniV32S16:
163 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 32, ScalarSizeInBits: 16) &&
164 MUI.isUniformAtDef(V: Reg);
165 case UniV32S32:
166 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 32, ScalarSizeInBits: 32) &&
167 MUI.isUniformAtDef(V: Reg);
168 case UniV2S64:
169 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64) &&
170 MUI.isUniformAtDef(V: Reg);
171 case UniB32:
172 return MRI.getType(Reg).getSizeInBits() == 32 && MUI.isUniformAtDef(V: Reg);
173 case UniB64:
174 return MRI.getType(Reg).getSizeInBits() == 64 && MUI.isUniformAtDef(V: Reg);
175 case UniB96:
176 return MRI.getType(Reg).getSizeInBits() == 96 && MUI.isUniformAtDef(V: Reg);
177 case UniB128:
178 return MRI.getType(Reg).getSizeInBits() == 128 && MUI.isUniformAtDef(V: Reg);
179 case UniB160:
180 return MRI.getType(Reg).getSizeInBits() == 160 && MUI.isUniformAtDef(V: Reg);
181 case UniB256:
182 return MRI.getType(Reg).getSizeInBits() == 256 && MUI.isUniformAtDef(V: Reg);
183 case UniB512:
184 return MRI.getType(Reg).getSizeInBits() == 512 && MUI.isUniformAtDef(V: Reg);
185 case UniBRC: {
186 if (MUI.isDivergentAtDef(V: Reg))
187 return false;
188 // Check if there is SGPR register class of same size as the LLT.
189 const SIRegisterInfo *TRI =
190 static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
191 // There is no 16 bit SGPR register class. Extra size check is required
192 // since getSGPRClassForBitWidth returns SReg_32RegClass for Size 16.
193 unsigned LLTSize = MRI.getType(Reg).getSizeInBits();
194 return LLTSize >= 32 && TRI->getSGPRClassForBitWidth(BitWidth: LLTSize);
195 }
196 case DivS1:
197 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 1) && MUI.isDivergentAtDef(V: Reg);
198 case DivS16:
199 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 16) && MUI.isDivergentAtDef(V: Reg);
200 case DivS32:
201 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 32) && MUI.isDivergentAtDef(V: Reg);
202 case DivS64:
203 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 64) && MUI.isDivergentAtDef(V: Reg);
204 case DivS128:
205 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 128) && MUI.isDivergentAtDef(V: Reg);
206 case DivP0:
207 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 0, SizeInBits: 64) && MUI.isDivergentAtDef(V: Reg);
208 case DivP1:
209 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 1, SizeInBits: 64) && MUI.isDivergentAtDef(V: Reg);
210 case DivP2:
211 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 2, SizeInBits: 32) && MUI.isDivergentAtDef(V: Reg);
212 case DivP3:
213 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 3, SizeInBits: 32) && MUI.isDivergentAtDef(V: Reg);
214 case DivP4:
215 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 4, SizeInBits: 64) && MUI.isDivergentAtDef(V: Reg);
216 case DivP5:
217 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 5, SizeInBits: 32) && MUI.isDivergentAtDef(V: Reg);
218 case DivPtr32:
219 return isAnyPtr(Ty: MRI.getType(Reg), Width: 32) && MUI.isDivergentAtDef(V: Reg);
220 case DivPtr64:
221 return isAnyPtr(Ty: MRI.getType(Reg), Width: 64) && MUI.isDivergentAtDef(V: Reg);
222 case DivPtr128:
223 return isAnyPtr(Ty: MRI.getType(Reg), Width: 128) && MUI.isDivergentAtDef(V: Reg);
224 case DivV2S16:
225 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16) &&
226 MUI.isDivergentAtDef(V: Reg);
227 case DivV2S32:
228 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32) &&
229 MUI.isDivergentAtDef(V: Reg);
230 case DivV4S32:
231 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32) &&
232 MUI.isDivergentAtDef(V: Reg);
233 case DivV2S64:
234 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64) &&
235 MUI.isDivergentAtDef(V: Reg);
236 case DivV3S32:
237 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 32) &&
238 MUI.isDivergentAtDef(V: Reg);
239 case DivV4S16:
240 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16) &&
241 MUI.isDivergentAtDef(V: Reg);
242 case DivV8S16:
243 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16) &&
244 MUI.isDivergentAtDef(V: Reg);
245 case DivV8S32:
246 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 32) &&
247 MUI.isDivergentAtDef(V: Reg);
248 case DivV16S16:
249 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 16) &&
250 MUI.isDivergentAtDef(V: Reg);
251 case DivV16S32:
252 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 32) &&
253 MUI.isDivergentAtDef(V: Reg);
254 case DivV6S32:
255 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 6, ScalarSizeInBits: 32) &&
256 MUI.isDivergentAtDef(V: Reg);
257 case DivV32S16:
258 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 32, ScalarSizeInBits: 16) &&
259 MUI.isDivergentAtDef(V: Reg);
260 case DivV32S32:
261 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 32, ScalarSizeInBits: 32) &&
262 MUI.isDivergentAtDef(V: Reg);
263 case DivB32:
264 return MRI.getType(Reg).getSizeInBits() == 32 && MUI.isDivergentAtDef(V: Reg);
265 case DivB64:
266 return MRI.getType(Reg).getSizeInBits() == 64 && MUI.isDivergentAtDef(V: Reg);
267 case DivB96:
268 return MRI.getType(Reg).getSizeInBits() == 96 && MUI.isDivergentAtDef(V: Reg);
269 case DivB128:
270 return MRI.getType(Reg).getSizeInBits() == 128 && MUI.isDivergentAtDef(V: Reg);
271 case DivB160:
272 return MRI.getType(Reg).getSizeInBits() == 160 && MUI.isDivergentAtDef(V: Reg);
273 case DivB256:
274 return MRI.getType(Reg).getSizeInBits() == 256 && MUI.isDivergentAtDef(V: Reg);
275 case DivB512:
276 return MRI.getType(Reg).getSizeInBits() == 512 && MUI.isDivergentAtDef(V: Reg);
277 case DivBRC: {
278 if (MUI.isUniformAtDef(V: Reg))
279 return false;
280 // Check if there is VGPR register class of same size as the LLT.
281 const SIRegisterInfo *TRI =
282 static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
283 return TRI->getSGPRClassForBitWidth(BitWidth: MRI.getType(Reg).getSizeInBits());
284 }
285 case BRC: {
286 // Check if there is SGPR and VGPR register class of same size as the LLT.
287 const SIRegisterInfo *TRI =
288 static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
289 unsigned LLTSize = MRI.getType(Reg).getSizeInBits();
290 return LLTSize >= 32 && TRI->getSGPRClassForBitWidth(BitWidth: LLTSize) &&
291 TRI->getVGPRClassForBitWidth(BitWidth: LLTSize);
292 }
293 case _:
294 return true;
295 default:
296 llvm_unreachable("missing matchUniformityAndLLT");
297 }
298}
299
300bool PredicateMapping::match(const MachineInstr &MI,
301 const MachineUniformityInfo &MUI,
302 const MachineRegisterInfo &MRI) const {
303 // Check LLT signature.
304 for (unsigned i = 0; i < OpUniformityAndTypes.size(); ++i) {
305 const MachineOperand &MO = MI.getOperand(i);
306 if (OpUniformityAndTypes[i] == _) {
307 assert((!MI.getOperand(i).isReg() ||
308 !MI.getOperand(i).getReg().isVirtual()) &&
309 "_ is for non-register and physical register operands only");
310 continue;
311 }
312
313 // Remaining IDs check registers.
314 if (!MO.isReg())
315 return false;
316
317 if (!matchUniformityAndLLT(Reg: MO.getReg(), UniID: OpUniformityAndTypes[i], MUI, MRI))
318 return false;
319 }
320
321 // More complex check.
322 if (TestFunc)
323 return TestFunc(MI);
324
325 return true;
326}
327
328SetOfRulesForOpcode::SetOfRulesForOpcode() = default;
329
330SetOfRulesForOpcode::SetOfRulesForOpcode(FastRulesTypes FastTypes)
331 : FastTypes(FastTypes) {}
332
333UniformityLLTOpPredicateID LLTToId(LLT Ty) {
334 if (Ty == LLT::scalar(SizeInBits: 16))
335 return S16;
336 if (Ty == LLT::scalar(SizeInBits: 32))
337 return S32;
338 if (Ty == LLT::scalar(SizeInBits: 64))
339 return S64;
340 if (Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16))
341 return V2S16;
342 if (Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32))
343 return V2S32;
344 if (Ty == LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 32))
345 return V3S32;
346 if (Ty == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32))
347 return V4S32;
348 return _;
349}
350
351UniformityLLTOpPredicateID LLTToBId(LLT Ty) {
352 if (Ty == LLT::scalar(SizeInBits: 32) || Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16) ||
353 isAnyPtr(Ty, Width: 32))
354 return B32;
355 if (Ty == LLT::scalar(SizeInBits: 64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32) ||
356 Ty == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16) || isAnyPtr(Ty, Width: 64))
357 return B64;
358 if (Ty == LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 32))
359 return B96;
360 if (Ty == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32) || Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64) ||
361 Ty == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16) || isAnyPtr(Ty, Width: 128))
362 return B128;
363 return _;
364}
365
366const RegBankLLTMapping *
367SetOfRulesForOpcode::findMappingForMI(const MachineInstr &MI,
368 const MachineRegisterInfo &MRI,
369 const MachineUniformityInfo &MUI) const {
370 // Search in "Fast Rules".
371 // Note: if fast rules are enabled, RegBankLLTMapping must be added in each
372 // slot that could "match fast Predicate". If not, InvalidMapping is
373 // returned which results in failure, does not search "Slow Rules".
374 if (FastTypes != NoFastRules) {
375 Register Reg = MI.getOperand(i: 0).getReg();
376 int Slot;
377 if (FastTypes == StandardB)
378 Slot = getFastPredicateSlot(Ty: LLTToBId(Ty: MRI.getType(Reg)));
379 else
380 Slot = getFastPredicateSlot(Ty: LLTToId(Ty: MRI.getType(Reg)));
381
382 if (Slot != -1)
383 return MUI.isUniformAtDef(V: Reg) ? &Uni[Slot] : &Div[Slot];
384 }
385
386 // Slow search for more complex rules.
387 for (const RegBankLegalizeRule &Rule : Rules) {
388 if (Rule.Predicate.match(MI, MUI, MRI))
389 return &Rule.OperandMapping;
390 }
391
392 return nullptr;
393}
394
395void SetOfRulesForOpcode::addRule(RegBankLegalizeRule Rule) {
396 Rules.push_back(Elt: Rule);
397}
398
399void SetOfRulesForOpcode::addFastRuleDivergent(UniformityLLTOpPredicateID Ty,
400 RegBankLLTMapping RuleApplyIDs) {
401 int Slot = getFastPredicateSlot(Ty);
402 assert(Slot != -1 && "Ty unsupported in this FastRulesTypes");
403 Div[Slot] = std::move(RuleApplyIDs);
404}
405
406void SetOfRulesForOpcode::addFastRuleUniform(UniformityLLTOpPredicateID Ty,
407 RegBankLLTMapping RuleApplyIDs) {
408 int Slot = getFastPredicateSlot(Ty);
409 assert(Slot != -1 && "Ty unsupported in this FastRulesTypes");
410 Uni[Slot] = std::move(RuleApplyIDs);
411}
412
413int SetOfRulesForOpcode::getFastPredicateSlot(
414 UniformityLLTOpPredicateID Ty) const {
415 switch (FastTypes) {
416 case Standard: {
417 switch (Ty) {
418 case S32:
419 return 0;
420 case S16:
421 return 1;
422 case S64:
423 return 2;
424 case V2S16:
425 return 3;
426 default:
427 return -1;
428 }
429 }
430 case StandardB: {
431 switch (Ty) {
432 case B32:
433 return 0;
434 case B64:
435 return 1;
436 case B96:
437 return 2;
438 case B128:
439 return 3;
440 default:
441 return -1;
442 }
443 }
444 case Vector: {
445 switch (Ty) {
446 case S32:
447 return 0;
448 case V2S32:
449 return 1;
450 case V3S32:
451 return 2;
452 case V4S32:
453 return 3;
454 default:
455 return -1;
456 }
457 }
458 default:
459 return -1;
460 }
461}
462
463RegBankLegalizeRules::RuleSetInitializer
464RegBankLegalizeRules::addRulesForGOpcs(std::initializer_list<unsigned> OpcList,
465 FastRulesTypes FastTypes) {
466 return RuleSetInitializer(OpcList, GRulesAlias, GRules, FastTypes);
467}
468
469RegBankLegalizeRules::RuleSetInitializer
470RegBankLegalizeRules::addRulesForIOpcs(std::initializer_list<unsigned> OpcList,
471 FastRulesTypes FastTypes) {
472 return RuleSetInitializer(OpcList, IRulesAlias, IRules, FastTypes);
473}
474
475const SetOfRulesForOpcode *
476RegBankLegalizeRules::getRulesForOpc(MachineInstr &MI) const {
477 unsigned Opc = MI.getOpcode();
478 if (Opc == AMDGPU::G_INTRINSIC || Opc == AMDGPU::G_INTRINSIC_CONVERGENT ||
479 Opc == AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS ||
480 Opc == AMDGPU::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS) {
481 unsigned IntrID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
482 auto IRAIt = IRulesAlias.find(Val: IntrID);
483 if (IRAIt == IRulesAlias.end())
484 return nullptr;
485 return &IRules.at(Val: IRAIt->second);
486 }
487
488 auto GRAIt = GRulesAlias.find(Val: Opc);
489 if (GRAIt == GRulesAlias.end())
490 return nullptr;
491 return &GRules.at(Val: GRAIt->second);
492}
493
494// Syntactic sugar wrapper for predicate lambda that enables '&&', '||' and '!'.
495class Predicate {
496private:
497 struct Elt {
498 // Save formula composed of Pred, '&&', '||' and '!' as a jump table.
499 // Sink ! to Pred. For example !((A && !B) || C) -> (!A || B) && !C
500 // Sequences of && and || will be represented by jumps, for example:
501 // (A && B && ... X) or (A && B && ... X) || Y
502 // A == true jump to B
503 // A == false jump to end or Y, result is A(false) or Y
504 // (A || B || ... X) or (A || B || ... X) && Y
505 // A == true jump to end or Y, result is A(true) or Y
506 // A == false jump to B
507 // Notice that when negating expression, we simply flip Neg on each Pred
508 // and swap TJumpOffset and FJumpOffset (&& becomes ||, || becomes &&).
509 std::function<bool(const MachineInstr &)> Pred;
510 bool Neg; // Neg of Pred is calculated before jump
511 unsigned TJumpOffset;
512 unsigned FJumpOffset;
513 };
514
515 SmallVector<Elt, 8> Expression;
516
517 Predicate(SmallVectorImpl<Elt> &&Expr) { Expression.swap(RHS&: Expr); };
518
519public:
520 Predicate(std::function<bool(const MachineInstr &)> Pred) {
521 Expression.push_back(Elt: {.Pred: Pred, .Neg: false, .TJumpOffset: 1, .FJumpOffset: 1});
522 };
523
524 bool operator()(const MachineInstr &MI) const {
525 unsigned Idx = 0;
526 unsigned ResultIdx = Expression.size();
527 bool Result;
528 do {
529 Result = Expression[Idx].Pred(MI);
530 Result = Expression[Idx].Neg ? !Result : Result;
531 if (Result) {
532 Idx += Expression[Idx].TJumpOffset;
533 } else {
534 Idx += Expression[Idx].FJumpOffset;
535 }
536 } while ((Idx != ResultIdx));
537
538 return Result;
539 };
540
541 Predicate operator!() const {
542 SmallVector<Elt, 8> NegExpression;
543 for (const Elt &ExprElt : Expression) {
544 NegExpression.push_back(Elt: {.Pred: ExprElt.Pred, .Neg: !ExprElt.Neg, .TJumpOffset: ExprElt.FJumpOffset,
545 .FJumpOffset: ExprElt.TJumpOffset});
546 }
547 return Predicate(std::move(NegExpression));
548 };
549
550 Predicate operator&&(const Predicate &RHS) const {
551 SmallVector<Elt, 8> AndExpression = Expression;
552
553 unsigned RHSSize = RHS.Expression.size();
554 unsigned ResultIdx = Expression.size();
555 for (unsigned i = 0; i < ResultIdx; ++i) {
556 // LHS results in false, whole expression results in false.
557 if (i + AndExpression[i].FJumpOffset == ResultIdx)
558 AndExpression[i].FJumpOffset += RHSSize;
559 }
560
561 AndExpression.append(RHS: RHS.Expression);
562
563 return Predicate(std::move(AndExpression));
564 }
565
566 Predicate operator||(const Predicate &RHS) const {
567 SmallVector<Elt, 8> OrExpression = Expression;
568
569 unsigned RHSSize = RHS.Expression.size();
570 unsigned ResultIdx = Expression.size();
571 for (unsigned i = 0; i < ResultIdx; ++i) {
572 // LHS results in true, whole expression results in true.
573 if (i + OrExpression[i].TJumpOffset == ResultIdx)
574 OrExpression[i].TJumpOffset += RHSSize;
575 }
576
577 OrExpression.append(RHS: RHS.Expression);
578
579 return Predicate(std::move(OrExpression));
580 }
581};
582
583// Initialize rules
584RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
585 MachineRegisterInfo &_MRI)
586 : ST(&_ST), MRI(&_MRI) {
587
588 addRulesForGOpcs(OpcList: {G_ADD, G_SUB}, FastTypes: Standard)
589 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr32Trunc}, {Sgpr32AExt, Sgpr32AExt}})
590 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
591 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}})
592 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
593 .Uni(Ty: V2S16, RuleApplyIDs: {{SgprV2S16}, {SgprV2S16, SgprV2S16}, UnpackAExt})
594 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
595 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {Sgpr64, Sgpr64}})
596 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr64}})
597 .Any(Init: {.Predicate: {UniV2S64}, .OperandMapping: {{UniInVgprV2S64}, {VgprV2S64, VgprV2S64}}})
598 .Any(Init: {.Predicate: {DivV2S64}, .OperandMapping: {{VgprV2S64}, {VgprV2S64, VgprV2S64}}});
599
600 addRulesForGOpcs(OpcList: {G_UADDO, G_USUBO}, FastTypes: Standard)
601 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32, Sgpr32Trunc}, {Sgpr32, Sgpr32}})
602 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32, Vcc}, {Vgpr32, Vgpr32}});
603
604 addRulesForGOpcs(OpcList: {G_UADDE, G_USUBE, G_SADDE, G_SSUBE}, FastTypes: Standard)
605 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32, Sgpr32Trunc}, {Sgpr32, Sgpr32, Sgpr32AExtBoolInReg}})
606 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32, Vcc}, {Vgpr32, Vgpr32, Vcc}});
607
608 addRulesForGOpcs(OpcList: {G_UADDSAT, G_SADDSAT, G_USUBSAT, G_SSUBSAT}, FastTypes: Standard)
609 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16, Vgpr16}})
610 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
611 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32}})
612 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
613 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16}})
614 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}});
615
616 bool HasVecMulU64 = ST->hasVMulU64Inst();
617 addRulesForGOpcs(OpcList: {G_MUL}, FastTypes: Standard)
618 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
619 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}})
620 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
621 .Uni(Ty: S64, RuleApplyIDs: {{SgprB64}, {SgprB64, SgprB64}})
622 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16}})
623 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
624 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr32Trunc}, {Sgpr32AExt, Sgpr32AExt}})
625 .Div(Ty: S64, RuleApplyIDs: {{VgprB64}, {VgprB64, VgprB64}}, STPred: HasVecMulU64)
626 .Div(Ty: S64, RuleApplyIDs: {{VgprB64}, {VgprB64, VgprB64}, SplitTo32Mul}, STPred: !HasVecMulU64);
627
628 bool hasMulHi = ST->hasScalarMulHiInsts();
629 addRulesForGOpcs(OpcList: {G_UMULH, G_SMULH}, FastTypes: Standard)
630 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
631 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}}, STPred: hasMulHi)
632 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32}}, STPred: !hasMulHi);
633
634 addRulesForGOpcs(OpcList: {G_AMDGPU_MAD_U64_U32}, FastTypes: Standard)
635 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64, Vcc}, {Vgpr32, Vgpr32, Vgpr64}})
636 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64, Sgpr32Trunc}, {Sgpr32, Sgpr32, Sgpr64}, UniMAD64});
637
638 bool HasScalarSMulU64 = ST->hasScalarSMulU64();
639 addRulesForGOpcs(OpcList: {G_AMDGPU_S_MUL_U64_U32, G_AMDGPU_S_MUL_I64_I32}, FastTypes: Standard)
640 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {Sgpr64, Sgpr64}, UniMul64}, STPred: HasScalarSMulU64)
641 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr64}, DivSMulToMAD});
642
643 addRulesForGOpcs(OpcList: {G_XOR, G_OR, G_AND}, FastTypes: StandardB)
644 .Any(Init: {.Predicate: {UniS1}, .OperandMapping: {{Sgpr32Trunc}, {Sgpr32AExt, Sgpr32AExt}}})
645 .Any(Init: {.Predicate: {DivS1}, .OperandMapping: {{Vcc}, {Vcc, Vcc}}})
646 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{Sgpr16}, {Sgpr16, Sgpr16}}})
647 .Any(Init: {.Predicate: {DivS16}, .OperandMapping: {{Vgpr16}, {Vgpr16, Vgpr16}}})
648 .Uni(Ty: B32, RuleApplyIDs: {{SgprB32}, {SgprB32, SgprB32}})
649 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {VgprB32, VgprB32}})
650 .Uni(Ty: B64, RuleApplyIDs: {{SgprB64}, {SgprB64, SgprB64}})
651 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {VgprB64, VgprB64}, SplitTo32});
652
653 addRulesForGOpcs(OpcList: {G_SHL}, FastTypes: Standard)
654 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr32Trunc}, {Sgpr32AExt, Sgpr32ZExt}})
655 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
656 .Uni(Ty: V2S16, RuleApplyIDs: {{SgprV2S16}, {SgprV2S16, SgprV2S16}, UnpackBitShift})
657 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
658 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}})
659 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {Sgpr64, Sgpr32}})
660 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
661 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr32}});
662
663 addRulesForGOpcs(OpcList: {G_LSHR}, FastTypes: Standard)
664 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr32Trunc}, {Sgpr32ZExt, Sgpr32ZExt}})
665 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
666 .Uni(Ty: V2S16, RuleApplyIDs: {{SgprV2S16}, {SgprV2S16, SgprV2S16}, UnpackBitShift})
667 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
668 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}})
669 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {Sgpr64, Sgpr32}})
670 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
671 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr32}});
672
673 addRulesForGOpcs(OpcList: {G_ASHR}, FastTypes: Standard)
674 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr32Trunc}, {Sgpr32SExt, Sgpr32ZExt}})
675 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
676 .Uni(Ty: V2S16, RuleApplyIDs: {{SgprV2S16}, {SgprV2S16, SgprV2S16}, UnpackBitShift})
677 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
678 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}})
679 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {Sgpr64, Sgpr32}})
680 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
681 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr32}});
682
683 addRulesForGOpcs(OpcList: {G_FSHR}, FastTypes: Standard)
684 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32, Vgpr32}})
685 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32, Vgpr32}});
686
687 addRulesForGOpcs(OpcList: {G_BSWAP}, FastTypes: Standard)
688 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16}})
689 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16}})
690 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32}})
691 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32}})
692 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {VgprV2S16}})
693 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16}});
694
695 addRulesForGOpcs(OpcList: {G_AMDGPU_CVT_F32_UBYTE0, G_AMDGPU_CVT_F32_UBYTE1,
696 G_AMDGPU_CVT_F32_UBYTE2, G_AMDGPU_CVT_F32_UBYTE3,
697 G_AMDGPU_RCP_IFLAG},
698 FastTypes: Standard)
699 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32}})
700 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32}});
701
702 addRulesForGOpcs(OpcList: {G_FRAME_INDEX}).Any(Init: {.Predicate: {UniP5, _}, .OperandMapping: {{SgprP5}, {None}}});
703
704 addRulesForGOpcs(OpcList: {G_UBFX, G_SBFX}, FastTypes: Standard)
705 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32, Sgpr32}, S_BFE})
706 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32, Vgpr32}})
707 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {Sgpr64, Sgpr32, Sgpr32}, S_BFE})
708 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr32, Vgpr32}, V_BFE});
709
710 addRulesForGOpcs(OpcList: {G_SMIN, G_SMAX}, FastTypes: Standard)
711 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr32Trunc}, {Sgpr32SExt, Sgpr32SExt}})
712 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
713 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}})
714 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
715 .Uni(Ty: V2S16, RuleApplyIDs: {{SgprV2S16}, {SgprV2S16, SgprV2S16}, UnpackMinMax})
716 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
717 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64, Vgpr64}})
718 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr64}});
719
720 addRulesForGOpcs(OpcList: {G_UMIN, G_UMAX}, FastTypes: Standard)
721 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr32Trunc}, {Sgpr32ZExt, Sgpr32ZExt}})
722 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
723 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}})
724 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
725 .Uni(Ty: V2S16, RuleApplyIDs: {{SgprV2S16}, {SgprV2S16, SgprV2S16}, UnpackMinMax})
726 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
727 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64, Vgpr64}})
728 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr64}});
729
730 addRulesForGOpcs(OpcList: {G_IMPLICIT_DEF})
731 .Any(Init: {.Predicate: {UniS1}, .OperandMapping: {{Sgpr32Trunc}, {}}})
732 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{Sgpr16}, {}}})
733 .Any(Init: {.Predicate: {UniBRC}, .OperandMapping: {{SgprBRC}, {}}});
734
735 addRulesForGOpcs(OpcList: {G_CONSTANT}, FastTypes: Standard)
736 .Any(Init: {.Predicate: {UniS1, _}, .OperandMapping: {{Sgpr32Trunc}, {}, UniCstExt}})
737 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {}})
738 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {}})
739 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {}})
740 .Any(Init: {.Predicate: {UniPtr32, _}, .OperandMapping: {{SgprPtr32}, {}}})
741 .Any(Init: {.Predicate: {UniPtr64, _}, .OperandMapping: {{SgprPtr64}, {}}});
742
743 addRulesForGOpcs(OpcList: {G_FCONSTANT}, FastTypes: Standard)
744 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {}})
745 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {}})
746 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {}});
747
748 addRulesForGOpcs(OpcList: {G_FREEZE})
749 .Any(Init: {.Predicate: {UniS1}, .OperandMapping: {{Sgpr32Trunc}, {Sgpr32AExt}}})
750 .Any(Init: {.Predicate: {DivS1}, .OperandMapping: {{Vcc}, {Vcc}}})
751 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{Sgpr16}, {Sgpr16}}})
752 .Any(Init: {.Predicate: {UniBRC}, .OperandMapping: {{SgprBRC}, {SgprBRC}}})
753 .Any(Init: {.Predicate: {DivBRC}, .OperandMapping: {{VgprBRC}, {VgprBRC}}});
754
755 addRulesForGOpcs(OpcList: {G_BITCAST})
756 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{Sgpr16}, {Sgpr16}}})
757 .Any(Init: {.Predicate: {DivS16}, .OperandMapping: {{Vgpr16}, {Vgpr16}}})
758 .Any(Init: {.Predicate: {UniBRC}, .OperandMapping: {{SgprBRC}, {SgprBRC}}})
759 .Any(Init: {.Predicate: {DivBRC}, .OperandMapping: {{VgprBRC}, {VgprBRC}}});
760
761 addRulesForGOpcs(OpcList: {G_UNMERGE_VALUES})
762 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{}, {}, UnmergeToShiftTrunc}})
763 .Any(Init: {.Predicate: {UniBRC}, .OperandMapping: {{}, {}, VerifyAllSgpr}})
764 .Any(Init: {.Predicate: {DivBRC}, .OperandMapping: {{}, {}, ApplyAllVgpr}});
765
766 addRulesForGOpcs(OpcList: {G_BUILD_VECTOR, G_MERGE_VALUES})
767 .Any(Init: {.Predicate: {UniBRC, S16}, .OperandMapping: {{}, {}, VerifyAllSgpr}})
768 .Any(Init: {.Predicate: {UniBRC, BRC}, .OperandMapping: {{}, {}, VerifyAllSgpr}})
769 .Any(Init: {.Predicate: {DivBRC, S16}, .OperandMapping: {{}, {}, ApplyAllVgpr}})
770 .Any(Init: {.Predicate: {DivBRC, BRC}, .OperandMapping: {{}, {}, ApplyAllVgpr}});
771
772 addRulesForGOpcs(OpcList: {G_CONCAT_VECTORS})
773 .Any(Init: {.Predicate: {UniBRC, BRC}, .OperandMapping: {{}, {}, VerifyAllSgpr}})
774 .Any(Init: {.Predicate: {DivBRC, BRC}, .OperandMapping: {{}, {}, ApplyAllVgpr}});
775
776 addRulesForGOpcs(OpcList: {G_PHI})
777 .Any(Init: {.Predicate: {UniS1}, .OperandMapping: {{}, {}, AextToS32InIncomingBlockGPHI}})
778 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{}, {}, VerifyAllSgprGPHI}})
779 .Any(Init: {.Predicate: {UniBRC}, .OperandMapping: {{}, {}, VerifyAllSgprGPHI}})
780 .Any(Init: {.Predicate: {DivBRC}, .OperandMapping: {{}, {}, VerifyAllSgprOrVgprGPHI}});
781
782 addRulesForGOpcs(OpcList: {G_EXTRACT_VECTOR_ELT})
783 .Any(Init: {.Predicate: {UniB32, UniBRC, UniS32}, .OperandMapping: {{SgprB32}, {SgprBRC, Sgpr32}}})
784 .Any(Init: {.Predicate: {DivB32, DivBRC, UniS32}, .OperandMapping: {{VgprB32}, {VgprBRC, Sgpr32}}})
785 .Any(Init: {.Predicate: {DivB32, BRC, DivS32},
786 .OperandMapping: {{VgprB32}, {VgprBRC, Vgpr32}, ExtrVecEltToSel}})
787 .Any(Init: {.Predicate: {UniB64, UniBRC, UniS32}, .OperandMapping: {{SgprB64}, {SgprBRC, Sgpr32}}})
788 .Any(Init: {.Predicate: {DivB64, DivBRC, UniS32},
789 .OperandMapping: {{VgprB64}, {VgprBRC, Sgpr32}, ExtrVecEltTo32}})
790 .Any(Init: {.Predicate: {DivB64, BRC, DivS32},
791 .OperandMapping: {{VgprB64}, {VgprBRC, Vgpr32}, ExtrVecEltToSel}});
792
793 addRulesForGOpcs(OpcList: {G_INSERT_VECTOR_ELT})
794 .Any(Init: {.Predicate: {UniBRC, UniBRC, UniB32, UniS32},
795 .OperandMapping: {{SgprBRC}, {SgprBRC, SgprB32, Sgpr32}}})
796 .Any(
797 Init: {.Predicate: {DivBRC, BRC, B32, UniS32}, .OperandMapping: {{VgprBRC}, {VgprBRC, VgprB32, Sgpr32}}})
798 .Any(Init: {.Predicate: {DivBRC, BRC, B32, DivS32},
799 .OperandMapping: {{VgprBRC}, {VgprBRC, VgprB32, Vgpr32}, InsVecEltToSel}})
800 .Any(Init: {.Predicate: {UniBRC, UniBRC, UniB64, UniS32},
801 .OperandMapping: {{SgprBRC}, {SgprBRC, SgprB64, Sgpr32}, InsVecEltToSel}})
802 .Any(Init: {.Predicate: {DivBRC, BRC, B64, UniS32},
803 .OperandMapping: {{VgprBRC}, {VgprBRC, VgprB64, Sgpr32}, InsVecEltTo32}})
804 .Any(Init: {.Predicate: {DivBRC, BRC, B64, DivS32},
805 .OperandMapping: {{VgprBRC}, {VgprBRC, VgprB64, Vgpr32}, InsVecEltToSel}});
806
807 // INTERSECT_RAY {Div}, {{VgprDst...}, {VgprSrc, ..., Sgpr_WF_RsrcIdx}}
808 // INTERSECT_RAY {Uni}, {{UniInVgprDst...}, {VgprSrc, ..., Sgpr_WF_RsrcIdx}}
809 addRulesForGOpcs(OpcList: {G_AMDGPU_BVH_INTERSECT_RAY, G_AMDGPU_BVH_DUAL_INTERSECT_RAY,
810 G_AMDGPU_BVH8_INTERSECT_RAY})
811 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {}, ApplyBVH_INTERSECT_RAY}});
812
813 // LOAD {Div}, {{VgprDst...}, {VgprSrc, ..., Sgpr_WF_RsrcIdx}}
814 // LOAD {Uni}, {{UniInVgprDst...}, {VgprSrc, ..., Sgpr_WF_RsrcIdx}}
815 // LOAD_NORET {}, {{}, {Imm, VgprSrc, ..., Sgpr_WF_RsrcIdx}}
816 // STORE {}, {{}, {VgprSrc, ..., Sgpr_WF_RsrcIdx}}
817 addRulesForGOpcs(OpcList: {G_AMDGPU_INTRIN_IMAGE_LOAD, G_AMDGPU_INTRIN_IMAGE_LOAD_D16,
818 G_AMDGPU_INTRIN_IMAGE_LOAD_NORET,
819 G_AMDGPU_INTRIN_IMAGE_STORE,
820 G_AMDGPU_INTRIN_IMAGE_STORE_D16})
821 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {}, ApplyINTRIN_IMAGE}});
822
823 Predicate isSignedICmp([](const MachineInstr &MI) -> bool {
824 auto Pred =
825 static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
826 return CmpInst::isSigned(Pred);
827 });
828
829 Predicate isEqualityICmp([](const MachineInstr &MI) -> bool {
830 auto Pred =
831 static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
832 return ICmpInst::isEquality(P: Pred);
833 });
834
835 bool HasScalarCompareEq64 = ST->hasScalarCompareEq64();
836 // clang-format off
837 addRulesForGOpcs(OpcList: {G_ICMP})
838 .Any(Init: {.Predicate: {{UniS1, _, S16}, isEqualityICmp}, .OperandMapping: {{Sgpr32Trunc}, {None, Sgpr32ZExt, Sgpr32ZExt}}})
839 .Any(Init: {.Predicate: {{UniS1, _, S16}, !isEqualityICmp && isSignedICmp}, .OperandMapping: {{Sgpr32Trunc}, {None, Sgpr32SExt, Sgpr32SExt}}})
840 .Any(Init: {.Predicate: {{UniS1, _, S16}, !isEqualityICmp && !isSignedICmp}, .OperandMapping: {{Sgpr32Trunc}, {None, Sgpr32ZExt, Sgpr32ZExt}}})
841 .Any(Init: {.Predicate: {{DivS1, _, S16}}, .OperandMapping: {{Vcc}, {None, Vgpr16, Vgpr16}}})
842 .Any(Init: {.Predicate: {{UniS1, _, S32}}, .OperandMapping: {{Sgpr32Trunc}, {None, Sgpr32, Sgpr32}}})
843 .Any(Init: {.Predicate: {{DivS1, _, S32}}, .OperandMapping: {{Vcc}, {None, Vgpr32, Vgpr32}}})
844 .Any(Init: {.Predicate: {{UniS1, _, S64}, isEqualityICmp}, .OperandMapping: {{Sgpr32Trunc}, {None, Sgpr64, Sgpr64}}}, STPred: HasScalarCompareEq64)
845 .Any(Init: {.Predicate: {{UniS1, _, S64}, isEqualityICmp}, .OperandMapping: {{UniInVcc}, {None, Vgpr64, Vgpr64}}}, STPred: !HasScalarCompareEq64)
846 .Any(Init: {.Predicate: {{UniS1, _, S64}, !isEqualityICmp}, .OperandMapping: {{UniInVcc}, {None, Vgpr64, Vgpr64}}})
847 .Any(Init: {.Predicate: {{DivS1, _, S64}}, .OperandMapping: {{Vcc}, {None, Vgpr64, Vgpr64}}})
848 .Any(Init: {.Predicate: {{UniS1, _, Ptr32}}, .OperandMapping: {{Sgpr32Trunc}, {None, SgprPtr32, SgprPtr32}}})
849 .Any(Init: {.Predicate: {{DivS1, _, Ptr32}}, .OperandMapping: {{Vcc}, {None, VgprPtr32, VgprPtr32}}})
850 .Any(Init: {.Predicate: {{UniS1, _, Ptr64}, isEqualityICmp}, .OperandMapping: {{Sgpr32Trunc}, {None, SgprPtr64, SgprPtr64}}}, STPred: HasScalarCompareEq64)
851 .Any(Init: {.Predicate: {{UniS1, _, Ptr64}, isEqualityICmp}, .OperandMapping: {{UniInVcc}, {None, VgprPtr64, VgprPtr64}}}, STPred: !HasScalarCompareEq64)
852 .Any(Init: {.Predicate: {{UniS1, _, Ptr64}, !isEqualityICmp}, .OperandMapping: {{UniInVcc}, {None, VgprPtr64, VgprPtr64}}})
853 .Any(Init: {.Predicate: {{DivS1, _, Ptr64}}, .OperandMapping: {{Vcc}, {None, VgprPtr64, VgprPtr64}}});
854 // clang-format on
855
856 addRulesForGOpcs(OpcList: {G_BRCOND})
857 .Any(Init: {.Predicate: {UniS1}, .OperandMapping: {{}, {Sgpr32AExtBoolInReg}}})
858 .Any(Init: {.Predicate: {DivS1}, .OperandMapping: {{}, {Vcc}}});
859
860 addRulesForGOpcs(OpcList: {G_BR}).Any(Init: {.Predicate: {_}, .OperandMapping: {{}, {None}}});
861
862 addRulesForGOpcs(OpcList: {G_SELECT}, FastTypes: StandardB)
863 .Any(Init: {.Predicate: {DivS16}, .OperandMapping: {{Vgpr16}, {Vcc, Vgpr16, Vgpr16}}})
864 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{Sgpr16}, {Sgpr32AExtBoolInReg, Sgpr16, Sgpr16}}})
865 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {Vcc, VgprB32, VgprB32}})
866 .Uni(Ty: B32, RuleApplyIDs: {{SgprB32}, {Sgpr32AExtBoolInReg, SgprB32, SgprB32}})
867 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {Vcc, VgprB64, VgprB64}, SplitTo32Select})
868 .Uni(Ty: B64, RuleApplyIDs: {{SgprB64}, {Sgpr32AExtBoolInReg, SgprB64, SgprB64}});
869
870 addRulesForGOpcs(OpcList: {G_ANYEXT})
871 .Any(Init: {.Predicate: {UniS16, S1}, .OperandMapping: {{None}, {None}}}) // should be combined away
872 .Any(Init: {.Predicate: {UniS32, S1}, .OperandMapping: {{None}, {None}}}) // should be combined away
873 .Any(Init: {.Predicate: {UniS64, S1}, .OperandMapping: {{None}, {None}}}) // should be combined away
874 .Any(Init: {.Predicate: {DivS16, S1}, .OperandMapping: {{Vgpr16}, {Vcc}, VccExtToSel}})
875 .Any(Init: {.Predicate: {DivS32, S1}, .OperandMapping: {{Vgpr32}, {Vcc}, VccExtToSel}})
876 .Any(Init: {.Predicate: {DivS64, S1}, .OperandMapping: {{Vgpr64}, {Vcc}, VccExtToSel}})
877 .Any(Init: {.Predicate: {UniS64, S32}, .OperandMapping: {{Sgpr64}, {Sgpr32}, Ext32To64}})
878 .Any(Init: {.Predicate: {DivS64, S32}, .OperandMapping: {{Vgpr64}, {Vgpr32}, Ext32To64}})
879 .Any(Init: {.Predicate: {UniS32, S16}, .OperandMapping: {{Sgpr32}, {Sgpr16}}})
880 .Any(Init: {.Predicate: {DivS32, S16}, .OperandMapping: {{Vgpr32}, {Vgpr16}}});
881
882 bool Has16bitCmp = ST->has16BitInsts();
883
884 // In global-isel G_TRUNC in-reg is treated as no-op, inst selected into COPY.
885 // It is up to user to deal with truncated bits.
886 // S1, S16, S32 and S64 results are handled with specific rules. Remaining
887 // (result, source) pairs with valid register classes are covered by the
888 // generic UniBRC/DivBRC wildcard rules.
889 addRulesForGOpcs(OpcList: {G_TRUNC})
890 .Any(Init: {.Predicate: {UniS1, UniS16}, .OperandMapping: {{None}, {None}}}) // should be combined away
891 .Any(Init: {.Predicate: {UniS1, UniS32}, .OperandMapping: {{None}, {None}}}) // should be combined away
892 .Any(Init: {.Predicate: {UniS1, UniS64}, .OperandMapping: {{None}, {None}}}) // should be combined away
893 .Any(Init: {.Predicate: {UniS16, S32}, .OperandMapping: {{Sgpr16}, {Sgpr32}}})
894 .Any(Init: {.Predicate: {UniBRC, UniBRC}, .OperandMapping: {{SgprBRC}, {SgprBRC}}})
895 .Any(Init: {.Predicate: {DivBRC, DivBRC}, .OperandMapping: {{VgprBRC}, {VgprBRC}}})
896 .Any(Init: {.Predicate: {UniV2S16, V2S32}, .OperandMapping: {{SgprV2S16}, {SgprV2S32}}})
897 .Any(Init: {.Predicate: {DivV2S16, V2S32}, .OperandMapping: {{VgprV2S16}, {VgprV2S32}}})
898 // This is non-trivial. VgprToVccCopy is done using compare instruction.
899 .Any(Init: {.Predicate: {DivS1, DivS16}, .OperandMapping: {{Vcc}, {Vgpr16}, VgprToVccCopy}}, STPred: Has16bitCmp)
900 .Any(Init: {.Predicate: {DivS1, DivS16}, .OperandMapping: {{Vcc}, {Vgpr32AExt}, VgprToVccCopy}},
901 STPred: !Has16bitCmp)
902 .Any(Init: {.Predicate: {DivS1, DivS32}, .OperandMapping: {{Vcc}, {Vgpr32}, VgprToVccCopy}})
903 .Any(Init: {.Predicate: {DivS1, DivS64}, .OperandMapping: {{Vcc}, {Vgpr64}, VgprToVccCopy}});
904
905 addRulesForGOpcs(OpcList: {G_ZEXT})
906 .Any(Init: {.Predicate: {UniS16, S1}, .OperandMapping: {{Sgpr32Trunc}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
907 .Any(Init: {.Predicate: {UniS32, S1}, .OperandMapping: {{Sgpr32}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
908 .Any(Init: {.Predicate: {UniS64, S1}, .OperandMapping: {{Sgpr64}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
909 .Any(Init: {.Predicate: {DivS16, S1}, .OperandMapping: {{Vgpr16}, {Vcc}, VccExtToSel}})
910 .Any(Init: {.Predicate: {DivS32, S1}, .OperandMapping: {{Vgpr32}, {Vcc}, VccExtToSel}})
911 .Any(Init: {.Predicate: {DivS64, S1}, .OperandMapping: {{Vgpr64}, {Vcc}, VccExtToSel}})
912 .Any(Init: {.Predicate: {UniS64, S32}, .OperandMapping: {{Sgpr64}, {Sgpr32}, Ext32To64}})
913 .Any(Init: {.Predicate: {DivS64, S32}, .OperandMapping: {{Vgpr64}, {Vgpr32}, Ext32To64}})
914 // not extending S16 to S32 is questionable.
915 .Any(Init: {.Predicate: {UniS64, S16}, .OperandMapping: {{Sgpr64}, {Sgpr32ZExt}, Ext32To64}})
916 .Any(Init: {.Predicate: {DivS64, S16}, .OperandMapping: {{Vgpr64}, {Vgpr32ZExt}, Ext32To64}})
917 .Any(Init: {.Predicate: {UniS32, S16}, .OperandMapping: {{Sgpr32}, {Sgpr16}}})
918 .Any(Init: {.Predicate: {DivS32, S16}, .OperandMapping: {{Vgpr32}, {Vgpr16}}});
919
920 addRulesForGOpcs(OpcList: {G_SEXT})
921 .Any(Init: {.Predicate: {UniS16, S1}, .OperandMapping: {{Sgpr32Trunc}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
922 .Any(Init: {.Predicate: {UniS32, S1}, .OperandMapping: {{Sgpr32}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
923 .Any(Init: {.Predicate: {UniS64, S1}, .OperandMapping: {{Sgpr64}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
924 .Any(Init: {.Predicate: {DivS16, S1}, .OperandMapping: {{Vgpr16}, {Vcc}, VccExtToSel}})
925 .Any(Init: {.Predicate: {DivS32, S1}, .OperandMapping: {{Vgpr32}, {Vcc}, VccExtToSel}})
926 .Any(Init: {.Predicate: {DivS64, S1}, .OperandMapping: {{Vgpr64}, {Vcc}, VccExtToSel}})
927 .Any(Init: {.Predicate: {UniS64, S32}, .OperandMapping: {{Sgpr64}, {Sgpr32}, Ext32To64}})
928 .Any(Init: {.Predicate: {DivS64, S32}, .OperandMapping: {{Vgpr64}, {Vgpr32}, Ext32To64}})
929 // not extending S16 to S32 is questionable.
930 .Any(Init: {.Predicate: {UniS64, S16}, .OperandMapping: {{Sgpr64}, {Sgpr32SExt}, Ext32To64}})
931 .Any(Init: {.Predicate: {DivS64, S16}, .OperandMapping: {{Vgpr64}, {Vgpr32SExt}, Ext32To64}})
932 .Any(Init: {.Predicate: {UniS32, S16}, .OperandMapping: {{Sgpr32}, {Sgpr16}}})
933 .Any(Init: {.Predicate: {DivS32, S16}, .OperandMapping: {{Vgpr32}, {Vgpr16}}});
934
935 addRulesForGOpcs(OpcList: {G_SEXT_INREG})
936 .Any(Init: {.Predicate: {UniS32, S32}, .OperandMapping: {{Sgpr32}, {Sgpr32}}})
937 .Any(Init: {.Predicate: {DivS32, S32}, .OperandMapping: {{Vgpr32}, {Vgpr32}}})
938 .Any(Init: {.Predicate: {UniS64, S64}, .OperandMapping: {{Sgpr64}, {Sgpr64}}})
939 .Any(Init: {.Predicate: {DivS64, S64}, .OperandMapping: {{Vgpr64}, {Vgpr64}, SplitTo32SExtInReg}});
940
941 addRulesForGOpcs(OpcList: {G_ASSERT_ZEXT, G_ASSERT_SEXT}, FastTypes: Standard)
942 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Imm}})
943 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Imm}})
944 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {Sgpr64, Imm}})
945 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Imm}});
946
947 addRulesForGOpcs(OpcList: {G_ASSERT_ALIGN}, FastTypes: Standard)
948 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32}})
949 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32}})
950 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {Sgpr64}})
951 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64}})
952 .Any(Init: {.Predicate: {UniPtr32}, .OperandMapping: {{SgprPtr32}, {SgprPtr32}}})
953 .Any(Init: {.Predicate: {DivPtr32}, .OperandMapping: {{VgprPtr32}, {VgprPtr32}}})
954 .Any(Init: {.Predicate: {UniPtr64}, .OperandMapping: {{SgprPtr64}, {SgprPtr64}}})
955 .Any(Init: {.Predicate: {DivPtr64}, .OperandMapping: {{VgprPtr64}, {VgprPtr64}}});
956
957 // Atomic read-modify-write operations: result and value are always VGPR,
958 // pointer varies by address space.
959 addRulesForGOpcs(OpcList: {G_ATOMICRMW_ADD, G_ATOMICRMW_SUB, G_ATOMICRMW_XCHG,
960 G_ATOMICRMW_AND, G_ATOMICRMW_OR, G_ATOMICRMW_XOR,
961 G_ATOMICRMW_MIN, G_ATOMICRMW_MAX, G_ATOMICRMW_UMIN,
962 G_ATOMICRMW_UMAX, G_ATOMICRMW_UINC_WRAP,
963 G_ATOMICRMW_UDEC_WRAP, G_ATOMICRMW_FMIN, G_ATOMICRMW_FMAX})
964 .Any(Init: {.Predicate: {DivS32, P0, S32}, .OperandMapping: {{Vgpr32}, {VgprP0, Vgpr32}}})
965 .Any(Init: {.Predicate: {DivS64, P0, S64}, .OperandMapping: {{Vgpr64}, {VgprP0, Vgpr64}}})
966 .Any(Init: {.Predicate: {DivS32, P1, S32}, .OperandMapping: {{Vgpr32}, {VgprP1, Vgpr32}}})
967 .Any(Init: {.Predicate: {DivS64, P1, S64}, .OperandMapping: {{Vgpr64}, {VgprP1, Vgpr64}}})
968 .Any(Init: {.Predicate: {DivS32, P3, S32}, .OperandMapping: {{Vgpr32}, {VgprP3, Vgpr32}}})
969 .Any(Init: {.Predicate: {DivS64, P3, S64}, .OperandMapping: {{Vgpr64}, {VgprP3, Vgpr64}}});
970
971 addRulesForGOpcs(OpcList: {G_ATOMICRMW_USUB_SAT, G_ATOMICRMW_USUB_COND})
972 .Any(Init: {.Predicate: {DivS32, P0}, .OperandMapping: {{Vgpr32}, {VgprP0, Vgpr32}}})
973 .Any(Init: {.Predicate: {DivS32, P1}, .OperandMapping: {{Vgpr32}, {VgprP1, Vgpr32}}})
974 .Any(Init: {.Predicate: {DivS32, P3}, .OperandMapping: {{Vgpr32}, {VgprP3, Vgpr32}}});
975
976 bool HasAtomicFlatPkAdd16Insts = ST->hasAtomicFlatPkAdd16Insts();
977 bool HasAtomicBufferGlobalPkAddF16Insts =
978 ST->hasAtomicBufferGlobalPkAddF16NoRtnInsts() ||
979 ST->hasAtomicBufferGlobalPkAddF16Insts();
980 bool HasAtomicDsPkAdd16Insts = ST->hasAtomicDsPkAdd16Insts();
981 addRulesForGOpcs(OpcList: {G_ATOMICRMW_FADD})
982 .Any(Init: {.Predicate: {DivS32, P0, S32}, .OperandMapping: {{Vgpr32}, {VgprP0, Vgpr32}}})
983 .Any(Init: {.Predicate: {DivS64, P0, S64}, .OperandMapping: {{Vgpr64}, {VgprP0, Vgpr64}}})
984 .Any(Init: {.Predicate: {DivS32, P1, S32}, .OperandMapping: {{Vgpr32}, {VgprP1, Vgpr32}}})
985 .Any(Init: {.Predicate: {DivS64, P1, S64}, .OperandMapping: {{Vgpr64}, {VgprP1, Vgpr64}}})
986 .Any(Init: {.Predicate: {DivS32, P3, S32}, .OperandMapping: {{Vgpr32}, {VgprP3, Vgpr32}}})
987 .Any(Init: {.Predicate: {DivS64, P3, S64}, .OperandMapping: {{Vgpr64}, {VgprP3, Vgpr64}}})
988 .Any(Init: {.Predicate: {DivV2S16, P0, V2S16}, .OperandMapping: {{VgprV2S16}, {VgprP0, VgprV2S16}}},
989 STPred: HasAtomicFlatPkAdd16Insts)
990 .Any(Init: {.Predicate: {DivV2S16, P1, V2S16}, .OperandMapping: {{VgprV2S16}, {VgprP1, VgprV2S16}}},
991 STPred: HasAtomicBufferGlobalPkAddF16Insts)
992 .Any(Init: {.Predicate: {DivV2S16, P3, V2S16}, .OperandMapping: {{VgprV2S16}, {VgprP3, VgprV2S16}}},
993 STPred: HasAtomicDsPkAdd16Insts);
994
995 addRulesForGOpcs(OpcList: {G_ATOMIC_CMPXCHG})
996 .Any(Init: {.Predicate: {DivS32, P2}, .OperandMapping: {{Vgpr32}, {VgprP2, Vgpr32, Vgpr32}}})
997 .Any(Init: {.Predicate: {DivS64, P2}, .OperandMapping: {{Vgpr64}, {VgprP2, Vgpr64, Vgpr64}}})
998 .Any(Init: {.Predicate: {DivS32, P3}, .OperandMapping: {{Vgpr32}, {VgprP3, Vgpr32, Vgpr32}}})
999 .Any(Init: {.Predicate: {DivS64, P3}, .OperandMapping: {{Vgpr64}, {VgprP3, Vgpr64, Vgpr64}}});
1000
1001 addRulesForGOpcs(OpcList: {G_AMDGPU_ATOMIC_CMPXCHG})
1002 .Any(Init: {.Predicate: {DivS32, P0}, .OperandMapping: {{Vgpr32}, {VgprP0, VgprV2S32}}})
1003 .Any(Init: {.Predicate: {DivS32, P1}, .OperandMapping: {{Vgpr32}, {VgprP1, VgprV2S32}}})
1004 .Any(Init: {.Predicate: {DivS64, P0}, .OperandMapping: {{Vgpr64}, {VgprP0, VgprV2S64}}})
1005 .Any(Init: {.Predicate: {DivS64, P1}, .OperandMapping: {{Vgpr64}, {VgprP1, VgprV2S64}}});
1006
1007 addRulesForGOpcs(OpcList: {G_AMDGPU_BUFFER_ATOMIC_CMPSWAP}, FastTypes: Standard)
1008 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32},
1009 {Vgpr32, Vgpr32, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1010 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64},
1011 {Vgpr64, Vgpr64, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
1012
1013 addRulesForGOpcs(OpcList: {G_AMDGPU_BUFFER_ATOMIC_ADD, G_AMDGPU_BUFFER_ATOMIC_AND,
1014 G_AMDGPU_BUFFER_ATOMIC_DEC, G_AMDGPU_BUFFER_ATOMIC_FMAX,
1015 G_AMDGPU_BUFFER_ATOMIC_FMIN, G_AMDGPU_BUFFER_ATOMIC_INC,
1016 G_AMDGPU_BUFFER_ATOMIC_OR, G_AMDGPU_BUFFER_ATOMIC_SMAX,
1017 G_AMDGPU_BUFFER_ATOMIC_SMIN, G_AMDGPU_BUFFER_ATOMIC_SUB,
1018 G_AMDGPU_BUFFER_ATOMIC_SWAP, G_AMDGPU_BUFFER_ATOMIC_UMAX,
1019 G_AMDGPU_BUFFER_ATOMIC_UMIN, G_AMDGPU_BUFFER_ATOMIC_XOR},
1020 FastTypes: Standard)
1021 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1022 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
1023
1024 bool hasSMRDx3 = ST->hasScalarDwordx3Loads();
1025 bool hasSMRDSmall = ST->hasScalarSubwordLoads();
1026 bool usesTrue16 = ST->useRealTrue16Insts();
1027
1028 Predicate isAlign16([](const MachineInstr &MI) -> bool {
1029 return (*MI.memoperands_begin())->getAlign() >= Align(16);
1030 });
1031
1032 Predicate isAlign4([](const MachineInstr &MI) -> bool {
1033 return (*MI.memoperands_begin())->getAlign() >= Align(4);
1034 });
1035
1036 Predicate isAtomicMMO([](const MachineInstr &MI) -> bool {
1037 return (*MI.memoperands_begin())->isAtomic();
1038 });
1039
1040 Predicate isUniMMO([](const MachineInstr &MI) -> bool {
1041 return AMDGPU::isUniformMMO(MMO: *MI.memoperands_begin());
1042 });
1043
1044 Predicate isConst([](const MachineInstr &MI) -> bool {
1045 // Address space in MMO be different then address space on pointer.
1046 const MachineMemOperand *MMO = *MI.memoperands_begin();
1047 const unsigned AS = MMO->getAddrSpace();
1048 return AS == AMDGPUAS::CONSTANT_ADDRESS ||
1049 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT;
1050 });
1051
1052 Predicate isVolatileMMO([](const MachineInstr &MI) -> bool {
1053 return (*MI.memoperands_begin())->isVolatile();
1054 });
1055
1056 Predicate isInvMMO([](const MachineInstr &MI) -> bool {
1057 return (*MI.memoperands_begin())->isInvariant();
1058 });
1059
1060 Predicate isNoClobberMMO([](const MachineInstr &MI) -> bool {
1061 return (*MI.memoperands_begin())->getFlags() & MONoClobber;
1062 });
1063
1064 Predicate isNaturalAligned([](const MachineInstr &MI) -> bool {
1065 const MachineMemOperand *MMO = *MI.memoperands_begin();
1066 return MMO->getAlign() >= Align(MMO->getSize().getValue());
1067 });
1068
1069 Predicate is8Or16BitMMO([](const MachineInstr &MI) -> bool {
1070 const MachineMemOperand *MMO = *MI.memoperands_begin();
1071 const unsigned MemSize = 8 * MMO->getSize().getValue();
1072 return MemSize == 16 || MemSize == 8;
1073 });
1074
1075 Predicate is32BitMMO([](const MachineInstr &MI) -> bool {
1076 const MachineMemOperand *MMO = *MI.memoperands_begin();
1077 return 8 * MMO->getSize().getValue() == 32;
1078 });
1079
1080 auto isUL = !isAtomicMMO && isUniMMO && (isConst || !isVolatileMMO) &&
1081 (isConst || isInvMMO || isNoClobberMMO);
1082
1083 // clang-format off
1084 // TODO: S32Dst, 16-bit any-extending load should not appear on True16 targets
1085 addRulesForGOpcs(OpcList: {G_LOAD})
1086 // flat, addrspace(0), never uniform - flat_load
1087 .Any(Init: {.Predicate: {DivS16, P0}, .OperandMapping: {{Vgpr16}, {VgprP0}}}, STPred: usesTrue16)
1088 .Any(Init: {.Predicate: {DivB32, P0}, .OperandMapping: {{VgprB32}, {VgprP0}}}) // 32-bit load, 8-bit and 16-bit any-extending load
1089 .Any(Init: {.Predicate: {DivB64, P0}, .OperandMapping: {{VgprB64}, {VgprP0}}})
1090 .Any(Init: {.Predicate: {DivB96, P0}, .OperandMapping: {{VgprB96}, {VgprP0}}})
1091 .Any(Init: {.Predicate: {DivB128, P0}, .OperandMapping: {{VgprB128}, {VgprP0}}})
1092
1093 // global, addrspace(1)
1094 // divergent - global_load
1095 .Any(Init: {.Predicate: {DivS16, P1}, .OperandMapping: {{Vgpr16}, {VgprP1}}}, STPred: usesTrue16)
1096 .Any(Init: {.Predicate: {DivB32, P1}, .OperandMapping: {{VgprB32}, {VgprP1}}}) //32-bit load, 8-bit and 16-bit any-extending load
1097 .Any(Init: {.Predicate: {DivB64, P1}, .OperandMapping: {{VgprB64}, {VgprP1}}})
1098 .Any(Init: {.Predicate: {DivB96, P1}, .OperandMapping: {{VgprB96}, {VgprP1}}})
1099 .Any(Init: {.Predicate: {DivB128, P1}, .OperandMapping: {{VgprB128}, {VgprP1}}})
1100 .Any(Init: {.Predicate: {DivB256, P1}, .OperandMapping: {{VgprB256}, {VgprP1}, SplitLoad}})
1101 .Any(Init: {.Predicate: {DivB512, P1}, .OperandMapping: {{VgprB512}, {VgprP1}, SplitLoad}})
1102
1103 // uniform - s_load
1104 .Any(Init: {.Predicate: {{UniS16, P1}, isNaturalAligned && isUL}, .OperandMapping: {{Sgpr32Trunc}, {SgprP1}}}, STPred: usesTrue16 && hasSMRDSmall) // s16 load
1105 .Any(Init: {.Predicate: {{UniS16, P1}, isAlign4 && isUL}, .OperandMapping: {{Sgpr32Trunc}, {SgprP1}, WidenMMOToS32}}, STPred: usesTrue16 && !hasSMRDSmall) // s16 load to 32-bit load
1106 .Any(Init: {.Predicate: {{UniB32, P1}, isNaturalAligned && isUL}, .OperandMapping: {{SgprB32}, {SgprP1}}}, STPred: hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load
1107 // TODO: SplitLoad when !isNaturalAligned && isUL and target hasSMRDSmall
1108 .Any(Init: {.Predicate: {{UniB32, P1}, is8Or16BitMMO && isAlign4 && isUL}, .OperandMapping: {{SgprB32}, {SgprP1}, WidenMMOToS32}}, STPred: !hasSMRDSmall) //8-bit and 16-bit any-extending load to 32-bit load
1109 .Any(Init: {.Predicate: {{UniB32, P1}, is32BitMMO && isAlign4 && isUL}, .OperandMapping: {{SgprB32}, {SgprP1}}}) //32-bit load
1110 .Any(Init: {.Predicate: {{UniB64, P1}, isAlign4 && isUL}, .OperandMapping: {{SgprB64}, {SgprP1}}})
1111 .Any(Init: {.Predicate: {{UniB96, P1}, isAlign16 && isUL}, .OperandMapping: {{SgprB96}, {SgprP1}, WidenLoad}}, STPred: !hasSMRDx3)
1112 .Any(Init: {.Predicate: {{UniB96, P1}, isAlign4 && !isAlign16 && isUL}, .OperandMapping: {{SgprB96}, {SgprP1}, SplitLoad}}, STPred: !hasSMRDx3)
1113 .Any(Init: {.Predicate: {{UniB96, P1}, isAlign4 && isUL}, .OperandMapping: {{SgprB96}, {SgprP1}}}, STPred: hasSMRDx3)
1114 .Any(Init: {.Predicate: {{UniB128, P1}, isAlign4 && isUL}, .OperandMapping: {{SgprB128}, {SgprP1}}})
1115 .Any(Init: {.Predicate: {{UniB256, P1}, isAlign4 && isUL}, .OperandMapping: {{SgprB256}, {SgprP1}}})
1116 .Any(Init: {.Predicate: {{UniB512, P1}, isAlign4 && isUL}, .OperandMapping: {{SgprB512}, {SgprP1}}})
1117
1118 // Uniform via global or buffer load, for example volatile or non-aligned
1119 // uniform load. Not using standard {{UniInVgprTy}, {VgprP1}} since it is
1120 // selected as global_load, use SgprP1 for pointer instead to match
1121 // patterns without flat-for-global, default for GFX7 and older.
1122 // -> +flat-for-global + {{UniInVgprTy}, {SgprP1}} - global_load
1123 // -> -flat-for-global + {{UniInVgprTy}, {SgprP1}} - buffer_load
1124 .Any(Init: {.Predicate: {{UniS16, P1}, !isNaturalAligned || !isUL}, .OperandMapping: {{UniInVgprS16}, {SgprP1}}}, STPred: usesTrue16 && hasSMRDSmall) // s16 load
1125 .Any(Init: {.Predicate: {{UniS16, P1}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprS16}, {SgprP1}}}, STPred: usesTrue16 && !hasSMRDSmall) // s16 load
1126 .Any(Init: {.Predicate: {{UniB32, P1}, !isNaturalAligned || !isUL}, .OperandMapping: {{UniInVgprB32}, {SgprP1}}}, STPred: hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load
1127 .Any(Init: {.Predicate: {{UniB32, P1}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB32}, {SgprP1}}}, STPred: !hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load
1128 .Any(Init: {.Predicate: {{UniB64, P1}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB64}, {SgprP1}}})
1129 .Any(Init: {.Predicate: {{UniB96, P1}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB96}, {SgprP1}}})
1130 .Any(Init: {.Predicate: {{UniB128, P1}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB128}, {SgprP1}}})
1131 .Any(Init: {.Predicate: {{UniB256, P1}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB256}, {SgprP1}, SplitLoad}})
1132 .Any(Init: {.Predicate: {{UniB512, P1}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB512}, {SgprP1}, SplitLoad}})
1133
1134 // local, addrspace(3) - ds_load
1135 .Any(Init: {.Predicate: {DivS16, P3}, .OperandMapping: {{Vgpr16}, {VgprP3}}}, STPred: usesTrue16)
1136 .Any(Init: {.Predicate: {DivB32, P3}, .OperandMapping: {{VgprB32}, {VgprP3}}}) // 32-bit load, 8-bit and 16-bit any-extending load
1137 .Any(Init: {.Predicate: {DivB64, P3}, .OperandMapping: {{VgprB64}, {VgprP3}}})
1138 .Any(Init: {.Predicate: {DivB96, P3}, .OperandMapping: {{VgprB96}, {VgprP3}}})
1139 .Any(Init: {.Predicate: {DivB128, P3}, .OperandMapping: {{VgprB128}, {VgprP3}}})
1140
1141 .Any(Init: {.Predicate: {UniS16, P3}, .OperandMapping: {{UniInVgprS16}, {SgprP3}}}, STPred: usesTrue16) // 16-bit load
1142 .Any(Init: {.Predicate: {UniB32, P3}, .OperandMapping: {{UniInVgprB32}, {VgprP3}}}) // 32-bit load, 8-bit and 16-bit any-extending load
1143 .Any(Init: {.Predicate: {UniB64, P3}, .OperandMapping: {{UniInVgprB64}, {VgprP3}}})
1144 .Any(Init: {.Predicate: {UniB96, P3}, .OperandMapping: {{UniInVgprB96}, {VgprP3}}})
1145 .Any(Init: {.Predicate: {UniB128, P3}, .OperandMapping: {{UniInVgprB128}, {VgprP3}}})
1146
1147 // constant, addrspace(4)
1148 // divergent - global_load
1149 .Any(Init: {.Predicate: {DivS16, P4}, .OperandMapping: {{Vgpr16}, {VgprP4}}}, STPred: usesTrue16)
1150 .Any(Init: {.Predicate: {DivB32, P4}, .OperandMapping: {{VgprB32}, {VgprP4}}}) //32-bit load, 8-bit and 16-bit any-extending load
1151 .Any(Init: {.Predicate: {DivB64, P4}, .OperandMapping: {{VgprB64}, {VgprP4}}})
1152 .Any(Init: {.Predicate: {DivB96, P4}, .OperandMapping: {{VgprB96}, {VgprP4}}})
1153 .Any(Init: {.Predicate: {DivB128, P4}, .OperandMapping: {{VgprB128}, {VgprP4}}})
1154 .Any(Init: {.Predicate: {DivB256, P4}, .OperandMapping: {{VgprB256}, {VgprP4}, SplitLoad}})
1155 .Any(Init: {.Predicate: {DivB512, P4}, .OperandMapping: {{VgprB512}, {VgprP4}, SplitLoad}})
1156
1157 // uniform - s_load
1158 .Any(Init: {.Predicate: {{UniS16, P4}, isNaturalAligned && isUL}, .OperandMapping: {{Sgpr32Trunc}, {SgprP4}}}, STPred: usesTrue16 && hasSMRDSmall) // s16 load
1159 .Any(Init: {.Predicate: {{UniS16, P4}, isAlign4 && isUL}, .OperandMapping: {{Sgpr32Trunc}, {SgprP4}, WidenMMOToS32}}, STPred: usesTrue16 && !hasSMRDSmall) // s16 load to 32-bit load
1160 .Any(Init: {.Predicate: {{UniB32, P4}, isNaturalAligned && isUL}, .OperandMapping: {{SgprB32}, {SgprP4}}}, STPred: hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load
1161 .Any(Init: {.Predicate: {{UniB32, P4}, is8Or16BitMMO && isAlign4 && isUL}, .OperandMapping: {{SgprB32}, {SgprP4}, WidenMMOToS32}}, STPred: !hasSMRDSmall) //8-bit and 16-bit any-extending load to 32-bit load
1162 .Any(Init: {.Predicate: {{UniB32, P4}, is32BitMMO && isAlign4 && isUL}, .OperandMapping: {{SgprB32}, {SgprP4}}}) //32-bit load
1163 .Any(Init: {.Predicate: {{UniB64, P4}, isAlign4 && isUL}, .OperandMapping: {{SgprB64}, {SgprP4}}})
1164 .Any(Init: {.Predicate: {{UniB96, P4}, isAlign16 && isUL}, .OperandMapping: {{SgprB96}, {SgprP4}, WidenLoad}}, STPred: !hasSMRDx3)
1165 .Any(Init: {.Predicate: {{UniB96, P4}, isAlign4 && !isAlign16 && isUL}, .OperandMapping: {{SgprB96}, {SgprP4}, SplitLoad}}, STPred: !hasSMRDx3)
1166 .Any(Init: {.Predicate: {{UniB96, P4}, isAlign4 && isUL}, .OperandMapping: {{SgprB96}, {SgprP4}}}, STPred: hasSMRDx3)
1167 .Any(Init: {.Predicate: {{UniB128, P4}, isAlign4 && isUL}, .OperandMapping: {{SgprB128}, {SgprP4}}})
1168 .Any(Init: {.Predicate: {{UniB256, P4}, isAlign4 && isUL}, .OperandMapping: {{SgprB256}, {SgprP4}}})
1169 .Any(Init: {.Predicate: {{UniB512, P4}, isAlign4 && isUL}, .OperandMapping: {{SgprB512}, {SgprP4}}})
1170
1171 // uniform in vgpr - global_load or buffer_load
1172 .Any(Init: {.Predicate: {{UniS16, P4}, !isNaturalAligned || !isUL}, .OperandMapping: {{UniInVgprS16}, {SgprP4}}}, STPred: usesTrue16 && hasSMRDSmall) // s16 load
1173 .Any(Init: {.Predicate: {{UniS16, P4}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprS16}, {SgprP4}}}, STPred: usesTrue16 && !hasSMRDSmall) // s16 load
1174 .Any(Init: {.Predicate: {{UniB32, P4}, !isNaturalAligned || !isUL}, .OperandMapping: {{UniInVgprB32}, {SgprP4}}}, STPred: hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load
1175 .Any(Init: {.Predicate: {{UniB32, P4}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB32}, {SgprP4}}}, STPred: !hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load
1176 .Any(Init: {.Predicate: {{UniB64, P4}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB64}, {SgprP4}}})
1177 .Any(Init: {.Predicate: {{UniB96, P4}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB96}, {SgprP4}}})
1178 .Any(Init: {.Predicate: {{UniB128, P4}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB128}, {SgprP4}}})
1179 .Any(Init: {.Predicate: {{UniB256, P4}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB256}, {SgprP4}, SplitLoad}})
1180 .Any(Init: {.Predicate: {{UniB512, P4}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB512}, {SgprP4}, SplitLoad}})
1181
1182 // private, addrspace(5), never uniform - scratch_load
1183 .Any(Init: {.Predicate: {DivS16, P5}, .OperandMapping: {{Vgpr16}, {VgprP5}}}, STPred: usesTrue16)
1184 .Any(Init: {.Predicate: {DivB32, P5}, .OperandMapping: {{VgprB32}, {VgprP5}}}) // 32-bit load, 8-bit and 16-bit any-extending load
1185 .Any(Init: {.Predicate: {DivB64, P5}, .OperandMapping: {{VgprB64}, {VgprP5}}})
1186 .Any(Init: {.Predicate: {DivB96, P5}, .OperandMapping: {{VgprB96}, {VgprP5}}})
1187 .Any(Init: {.Predicate: {DivB128, P5}, .OperandMapping: {{VgprB128}, {VgprP5}}})
1188
1189 .Any(Init: {.Predicate: {DivS32, Ptr128}, .OperandMapping: {{Vgpr32}, {VgprPtr128}}});
1190
1191
1192 addRulesForGOpcs(OpcList: {G_ZEXTLOAD, G_SEXTLOAD}) // i8 and i16 zeroextending loads
1193 .Any(Init: {.Predicate: {DivS32, P0}, .OperandMapping: {{Vgpr32}, {VgprP0}}})
1194 .Any(Init: {.Predicate: {DivS16, P0}, .OperandMapping: {{Vgpr16}, {VgprP0}}}, STPred: usesTrue16)
1195
1196 .Any(Init: {.Predicate: {DivS32, P1}, .OperandMapping: {{Vgpr32}, {VgprP1}}})
1197 .Any(Init: {.Predicate: {DivS16, P1}, .OperandMapping: {{Vgpr16}, {VgprP1}}}, STPred: usesTrue16)
1198 .Any(Init: {.Predicate: {{UniS32, P1}, isAlign4 && isUL}, .OperandMapping: {{Sgpr32}, {SgprP1}, WidenMMOToS32}}, STPred: !hasSMRDSmall)
1199 .Any(Init: {.Predicate: {{UniS32, P1}, isNaturalAligned && isUL}, .OperandMapping: {{Sgpr32}, {SgprP1}}}, STPred: hasSMRDSmall)
1200 .Any(Init: {.Predicate: {{UniS32, P1}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprS32}, {SgprP1}}}, STPred: !hasSMRDSmall)
1201 .Any(Init: {.Predicate: {{UniS32, P1}, !isNaturalAligned || !isUL}, .OperandMapping: {{UniInVgprS32}, {SgprP1}}}, STPred: hasSMRDSmall)
1202
1203 .Any(Init: {.Predicate: {DivS32, P3}, .OperandMapping: {{Vgpr32}, {VgprP3}}})
1204 .Any(Init: {.Predicate: {DivS16, P3}, .OperandMapping: {{Vgpr16}, {VgprP3}}}, STPred: usesTrue16)
1205 .Any(Init: {.Predicate: {UniS32, P3}, .OperandMapping: {{UniInVgprS32}, {VgprP3}}})
1206
1207 .Any(Init: {.Predicate: {DivS32, P4}, .OperandMapping: {{Vgpr32}, {VgprP4}}})
1208 .Any(Init: {.Predicate: {DivS16, P4}, .OperandMapping: {{Vgpr16}, {VgprP4}}}, STPred: usesTrue16)
1209 .Any(Init: {.Predicate: {{UniS32, P4}, isAlign4 && isUL}, .OperandMapping: {{Sgpr32}, {SgprP4}, WidenMMOToS32}}, STPred: !hasSMRDSmall)
1210 .Any(Init: {.Predicate: {{UniS32, P4}, isNaturalAligned && isUL}, .OperandMapping: {{Sgpr32}, {SgprP4}}}, STPred: hasSMRDSmall)
1211 .Any(Init: {.Predicate: {{UniS32, P4}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprS32}, {SgprP4}}}, STPred: !hasSMRDSmall)
1212 .Any(Init: {.Predicate: {{UniS32, P4}, !isNaturalAligned || !isUL}, .OperandMapping: {{UniInVgprS32}, {SgprP4}}}, STPred: hasSMRDSmall)
1213
1214 .Any(Init: {.Predicate: {DivS32, P5}, .OperandMapping: {{Vgpr32}, {VgprP5}}})
1215 .Any(Init: {.Predicate: {DivS16, P5}, .OperandMapping: {{Vgpr16}, {VgprP5}}}, STPred: usesTrue16);
1216
1217 addRulesForGOpcs(OpcList: {G_STORE})
1218 // addrspace(0)
1219 .Any(Init: {.Predicate: {S16, P0}, .OperandMapping: {{}, {Vgpr16, VgprP0}}}, STPred: usesTrue16) // 16-bit store
1220 .Any(Init: {.Predicate: {B32, P0}, .OperandMapping: {{}, {VgprB32, VgprP0}}}) // 32-bit store, 8-bit and 16-bit truncating store
1221 .Any(Init: {.Predicate: {B64, P0}, .OperandMapping: {{}, {VgprB64, VgprP0}}})
1222 .Any(Init: {.Predicate: {B96, P0}, .OperandMapping: {{}, {VgprB96, VgprP0}}})
1223 .Any(Init: {.Predicate: {B128, P0}, .OperandMapping: {{}, {VgprB128, VgprP0}}})
1224
1225 // addrspace(1), there are no stores to addrspace(4)
1226 // For targets:
1227 // - with "+flat-for-global" - global_store
1228 // - without(-flat-for-global) - buffer_store addr64
1229 .Any(Init: {.Predicate: {S16, DivP1}, .OperandMapping: {{}, {Vgpr16, VgprP1}}}, STPred: usesTrue16) // 16-bit store
1230 .Any(Init: {.Predicate: {B32, DivP1}, .OperandMapping: {{}, {VgprB32, VgprP1}}}) // 32-bit store, 8-bit and 16-bit truncating store
1231 .Any(Init: {.Predicate: {B64, DivP1}, .OperandMapping: {{}, {VgprB64, VgprP1}}})
1232 .Any(Init: {.Predicate: {B96, DivP1}, .OperandMapping: {{}, {VgprB96, VgprP1}}})
1233 .Any(Init: {.Predicate: {B128, DivP1}, .OperandMapping: {{}, {VgprB128, VgprP1}}})
1234
1235 // For UniP1, use sgpr ptr to match flat-for-global patterns. Targets:
1236 // - with "+flat-for-global" - global_store for both sgpr and vgpr ptr
1237 // - without(-flat-for-global) - need sgpr ptr to select buffer_store
1238 .Any(Init: {.Predicate: {S16, UniP1}, .OperandMapping: {{}, {Vgpr16, SgprP1}}}, STPred: usesTrue16) // 16-bit store
1239 .Any(Init: {.Predicate: {B32, UniP1}, .OperandMapping: {{}, {VgprB32, SgprP1}}}) // 32-bit store, 8-bit and 16-bit truncating store
1240 .Any(Init: {.Predicate: {B64, UniP1}, .OperandMapping: {{}, {VgprB64, SgprP1}}})
1241 .Any(Init: {.Predicate: {B96, UniP1}, .OperandMapping: {{}, {VgprB96, SgprP1}}})
1242 .Any(Init: {.Predicate: {B128, UniP1}, .OperandMapping: {{}, {VgprB128, SgprP1}}})
1243
1244 // addrspace(3) and addrspace(5)
1245 .Any(Init: {.Predicate: {S16, Ptr32}, .OperandMapping: {{}, {Vgpr16, VgprPtr32}}}, STPred: usesTrue16) // 16-bit store
1246 .Any(Init: {.Predicate: {B32, Ptr32}, .OperandMapping: {{}, {VgprB32, VgprPtr32}}}) // 32-bit store, 8-bit and 16-bit truncating store
1247 .Any(Init: {.Predicate: {B64, Ptr32}, .OperandMapping: {{}, {VgprB64, VgprPtr32}}})
1248 .Any(Init: {.Predicate: {B96, Ptr32}, .OperandMapping: {{}, {VgprB96, VgprPtr32}}})
1249 .Any(Init: {.Predicate: {B128, Ptr32}, .OperandMapping: {{}, {VgprB128, VgprPtr32}}});
1250
1251 // clang-format on
1252
1253 addRulesForGOpcs(OpcList: {G_AMDGPU_BUFFER_LOAD, G_AMDGPU_BUFFER_LOAD_FORMAT,
1254 G_AMDGPU_TBUFFER_LOAD_FORMAT},
1255 FastTypes: StandardB)
1256 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1257 .Uni(Ty: B32, RuleApplyIDs: {{UniInVgprB32}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1258 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1259 .Uni(Ty: B64, RuleApplyIDs: {{UniInVgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1260 .Div(Ty: B96, RuleApplyIDs: {{VgprB96}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1261 .Uni(Ty: B96, RuleApplyIDs: {{UniInVgprB96}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1262 .Div(Ty: B128, RuleApplyIDs: {{VgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1263 .Uni(Ty: B128, RuleApplyIDs: {{UniInVgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
1264
1265 addRulesForGOpcs(OpcList: {G_AMDGPU_BUFFER_LOAD_USHORT, G_AMDGPU_BUFFER_LOAD_UBYTE,
1266 G_AMDGPU_BUFFER_LOAD_SSHORT, G_AMDGPU_BUFFER_LOAD_SBYTE},
1267 FastTypes: StandardB)
1268 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1269 .Uni(Ty: B32, RuleApplyIDs: {{UniInVgprB32}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
1270
1271 addRulesForGOpcs(
1272 OpcList: {G_AMDGPU_BUFFER_LOAD_UBYTE_TFE, G_AMDGPU_BUFFER_LOAD_USHORT_TFE},
1273 FastTypes: StandardB)
1274 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1275 .Uni(Ty: B64, RuleApplyIDs: {{UniInVgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
1276
1277 addRulesForGOpcs(OpcList: {G_AMDGPU_BUFFER_LOAD_TFE, G_AMDGPU_BUFFER_LOAD_FORMAT_TFE},
1278 FastTypes: StandardB)
1279 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1280 .Uni(Ty: B64, RuleApplyIDs: {{UniInVgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1281 .Div(Ty: B96, RuleApplyIDs: {{VgprB96}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1282 .Uni(Ty: B96, RuleApplyIDs: {{UniInVgprB96}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1283 .Div(Ty: B128, RuleApplyIDs: {{VgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1284 .Uni(Ty: B128, RuleApplyIDs: {{UniInVgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1285 .Any(Init: {.Predicate: {DivB160}, .OperandMapping: {{VgprB160}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}})
1286 .Any(Init: {.Predicate: {UniB160},
1287 .OperandMapping: {{UniInVgprB160}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}});
1288
1289 addRulesForGOpcs(
1290 OpcList: {G_AMDGPU_BUFFER_LOAD_FORMAT_D16, G_AMDGPU_TBUFFER_LOAD_FORMAT_D16},
1291 FastTypes: StandardB)
1292 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1293 .Uni(Ty: B32, RuleApplyIDs: {{UniInVgprB32}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1294 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1295 .Uni(Ty: B64, RuleApplyIDs: {{UniInVgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1296 .Div(Ty: B128, RuleApplyIDs: {{VgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1297 .Uni(Ty: B128, RuleApplyIDs: {{UniInVgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
1298
1299 addRulesForGOpcs(OpcList: {G_AMDGPU_S_BUFFER_LOAD})
1300 // waterfall expansion is part of S_BUF_to_BUF
1301 .Any(Init: {.Predicate: {UniB32}, .OperandMapping: {{SgprB32}, {SgprV4S32, Sgpr32}}})
1302 .Any(Init: {.Predicate: {DivB32, UniV4S32, DivB32},
1303 .OperandMapping: {{VgprB32}, {SgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1304 .Any(Init: {.Predicate: {DivB32, DivV4S32, UniB32},
1305 .OperandMapping: {{VgprB32}, {VgprV4S32, Sgpr32}, S_BUF_to_BUF}})
1306 .Any(Init: {.Predicate: {DivB32, DivV4S32, DivB32},
1307 .OperandMapping: {{VgprB32}, {VgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1308
1309 .Any(Init: {.Predicate: {UniB64}, .OperandMapping: {{SgprB64}, {SgprV4S32, Sgpr32}}})
1310 .Any(Init: {.Predicate: {DivB64, UniV4S32, DivB32},
1311 .OperandMapping: {{VgprB64}, {SgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1312 .Any(Init: {.Predicate: {DivB64, DivV4S32, UniB32},
1313 .OperandMapping: {{VgprB64}, {VgprV4S32, Sgpr32}, S_BUF_to_BUF}})
1314 .Any(Init: {.Predicate: {DivB64, DivV4S32, DivB32},
1315 .OperandMapping: {{VgprB64}, {VgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1316
1317 .Any(Init: {.Predicate: {UniB96}, .OperandMapping: {{SgprB96}, {SgprV4S32, Sgpr32}}})
1318 .Any(Init: {.Predicate: {DivB96, UniV4S32, DivB32},
1319 .OperandMapping: {{VgprB96}, {SgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1320 .Any(Init: {.Predicate: {DivB96, DivV4S32, UniB32},
1321 .OperandMapping: {{VgprB96}, {VgprV4S32, Sgpr32}, S_BUF_to_BUF}})
1322 .Any(Init: {.Predicate: {DivB96, DivV4S32, DivB32},
1323 .OperandMapping: {{VgprB96}, {VgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1324
1325 .Any(Init: {.Predicate: {UniB128}, .OperandMapping: {{SgprB128}, {SgprV4S32, Sgpr32}}})
1326 .Any(Init: {.Predicate: {DivB128, UniV4S32, DivB32},
1327 .OperandMapping: {{VgprB128}, {SgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1328 .Any(Init: {.Predicate: {DivB128, DivV4S32, UniB32},
1329 .OperandMapping: {{VgprB128}, {VgprV4S32, Sgpr32}, S_BUF_to_BUF}})
1330 .Any(Init: {.Predicate: {DivB128, DivV4S32, DivB32},
1331 .OperandMapping: {{VgprB128}, {VgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1332
1333 .Any(Init: {.Predicate: {UniB256}, .OperandMapping: {{SgprB256}, {SgprV4S32, Sgpr32}}})
1334 .Any(Init: {.Predicate: {DivB256, UniV4S32, DivB32},
1335 .OperandMapping: {{VgprB256}, {SgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1336 .Any(Init: {.Predicate: {DivB256, DivV4S32, UniB32},
1337 .OperandMapping: {{VgprB256}, {VgprV4S32, Sgpr32}, S_BUF_to_BUF}})
1338 .Any(Init: {.Predicate: {DivB256, DivV4S32, DivB32},
1339 .OperandMapping: {{VgprB256}, {VgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1340
1341 .Any(Init: {.Predicate: {UniB512}, .OperandMapping: {{SgprB512}, {SgprV4S32, Sgpr32}}})
1342 .Any(Init: {.Predicate: {DivB512, UniV4S32, DivB32},
1343 .OperandMapping: {{VgprB512}, {SgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1344 .Any(Init: {.Predicate: {DivB512, DivV4S32, UniB32},
1345 .OperandMapping: {{VgprB512}, {VgprV4S32, Sgpr32}, S_BUF_to_BUF}})
1346 .Any(Init: {.Predicate: {DivB512, DivV4S32, DivB32},
1347 .OperandMapping: {{VgprB512}, {VgprV4S32, Vgpr32}, S_BUF_to_BUF}});
1348
1349 addRulesForGOpcs(OpcList: {G_AMDGPU_S_BUFFER_LOAD_SBYTE, G_AMDGPU_S_BUFFER_LOAD_UBYTE,
1350 G_AMDGPU_S_BUFFER_LOAD_SSHORT,
1351 G_AMDGPU_S_BUFFER_LOAD_USHORT})
1352 .Any(Init: {.Predicate: {UniS32, UniV4S32, UniS32}, .OperandMapping: {{Sgpr32}, {SgprV4S32, Sgpr32}}})
1353 .Any(Init: {.Predicate: {DivS32, UniV4S32, DivS32},
1354 .OperandMapping: {{Vgpr32}, {SgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1355 .Any(Init: {.Predicate: {DivS32, DivV4S32, UniS32},
1356 .OperandMapping: {{Vgpr32}, {VgprV4S32, Sgpr32}, S_BUF_to_BUF}})
1357 .Any(Init: {.Predicate: {DivS32, DivV4S32, DivS32},
1358 .OperandMapping: {{Vgpr32}, {VgprV4S32, Vgpr32}, S_BUF_to_BUF}});
1359
1360 addRulesForGOpcs(OpcList: {G_AMDGPU_BUFFER_STORE, G_AMDGPU_BUFFER_STORE_BYTE,
1361 G_AMDGPU_BUFFER_STORE_SHORT, G_AMDGPU_BUFFER_STORE_FORMAT,
1362 G_AMDGPU_BUFFER_STORE_FORMAT_D16,
1363 G_AMDGPU_TBUFFER_STORE_FORMAT,
1364 G_AMDGPU_TBUFFER_STORE_FORMAT_D16})
1365 .Any(Init: {.Predicate: {B32}, .OperandMapping: {{}, {VgprB32, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}})
1366 .Any(Init: {.Predicate: {B64}, .OperandMapping: {{}, {VgprB64, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}})
1367 .Any(Init: {.Predicate: {B96}, .OperandMapping: {{}, {VgprB96, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}})
1368 .Any(Init: {.Predicate: {B128}, .OperandMapping: {{}, {VgprB128, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}});
1369
1370 // Buffer atomics: resource descriptor + scalar offset are SGPR, data and
1371 // address components are VGPR.
1372 //
1373 // Operand order (SIInstructions.td BufferAtomicGenericInstruction):
1374 // dst = op vdata, rsrc, vindex, voffset, soffset, offset_imm, cachepolicy,
1375 // idxen_imm
1376 addRulesForGOpcs(OpcList: {G_AMDGPU_BUFFER_ATOMIC_FADD})
1377 .Any(Init: {.Predicate: {S32, S32, V4S32, S32, S32, S32},
1378 .OperandMapping: {{Vgpr32}, {Vgpr32, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}})
1379 .Any(Init: {.Predicate: {S64, S64, V4S32, S32, S32, S32},
1380 .OperandMapping: {{Vgpr64}, {Vgpr64, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}})
1381 .Any(Init: {.Predicate: {V2S16, V2S16, V4S32, S32, S32, S32},
1382 .OperandMapping: {{VgprV2S16},
1383 {VgprV2S16, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}});
1384
1385 addRulesForGOpcs(OpcList: {G_PTR_ADD})
1386 .Any(Init: {.Predicate: {UniPtr32}, .OperandMapping: {{SgprPtr32}, {SgprPtr32, Sgpr32}}})
1387 .Any(Init: {.Predicate: {DivPtr32}, .OperandMapping: {{VgprPtr32}, {VgprPtr32, Vgpr32}}})
1388 .Any(Init: {.Predicate: {UniPtr64}, .OperandMapping: {{SgprPtr64}, {SgprPtr64, Sgpr64}}})
1389 .Any(Init: {.Predicate: {DivPtr64}, .OperandMapping: {{VgprPtr64}, {VgprPtr64, Vgpr64}}});
1390
1391 addRulesForGOpcs(OpcList: {G_INTTOPTR})
1392 .Any(Init: {.Predicate: {UniPtr32}, .OperandMapping: {{SgprPtr32}, {Sgpr32}}})
1393 .Any(Init: {.Predicate: {DivPtr32}, .OperandMapping: {{VgprPtr32}, {Vgpr32}}})
1394 .Any(Init: {.Predicate: {UniPtr64}, .OperandMapping: {{SgprPtr64}, {Sgpr64}}})
1395 .Any(Init: {.Predicate: {DivPtr64}, .OperandMapping: {{VgprPtr64}, {Vgpr64}}})
1396 .Any(Init: {.Predicate: {UniPtr128}, .OperandMapping: {{SgprPtr128}, {Sgpr128}}})
1397 .Any(Init: {.Predicate: {DivPtr128}, .OperandMapping: {{VgprPtr128}, {Vgpr128}}});
1398
1399 addRulesForGOpcs(OpcList: {G_PTRTOINT})
1400 .Any(Init: {.Predicate: {UniS32}, .OperandMapping: {{Sgpr32}, {SgprPtr32}}})
1401 .Any(Init: {.Predicate: {DivS32}, .OperandMapping: {{Vgpr32}, {VgprPtr32}}})
1402 .Any(Init: {.Predicate: {UniS64}, .OperandMapping: {{Sgpr64}, {SgprPtr64}}})
1403 .Any(Init: {.Predicate: {DivS64}, .OperandMapping: {{Vgpr64}, {VgprPtr64}}})
1404 .Any(Init: {.Predicate: {UniS128}, .OperandMapping: {{Sgpr128}, {SgprPtr128}}})
1405 .Any(Init: {.Predicate: {DivS128}, .OperandMapping: {{Vgpr128}, {VgprPtr128}}});
1406
1407 // FIXME: Update llvm/test/CodeGen/AMDGPU/ptrmask.ll to use GlobalISel.
1408 // Currently crashes on P8 (buffer resource) tests due to legalizer issue.
1409 addRulesForGOpcs(OpcList: {G_PTRMASK})
1410 .Any(Init: {.Predicate: {UniP1}, .OperandMapping: {{SgprP1}, {SgprP1, Sgpr64}}})
1411 .Any(Init: {.Predicate: {DivP1}, .OperandMapping: {{VgprP1}, {VgprP1, Vgpr64}}})
1412 .Any(Init: {.Predicate: {UniP3}, .OperandMapping: {{SgprP3}, {SgprP3, Sgpr32}}})
1413 .Any(Init: {.Predicate: {DivP3}, .OperandMapping: {{VgprP3}, {VgprP3, Vgpr32}}});
1414
1415 addRulesForGOpcs(OpcList: {G_DYN_STACKALLOC})
1416 .Any(Init: {.Predicate: {UniP5, UniS32}, .OperandMapping: {{SgprP5}, {Sgpr32}, DynStackAlloc}})
1417 .Any(Init: {.Predicate: {UniP5, DivS32}, .OperandMapping: {{SgprP5}, {Vgpr32}, DynStackAlloc}});
1418
1419 addRulesForGOpcs(OpcList: {G_ABS}, FastTypes: Standard)
1420 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr32Trunc}, {Sgpr32SExt}})
1421 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16}, AbsToNegMax})
1422 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32}})
1423 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32}, AbsToNegMax})
1424 .Uni(Ty: V2S16, RuleApplyIDs: {{SgprV2S16}, {SgprV2S16}, AbsToS32})
1425 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16}, AbsToNegMax});
1426
1427 addRulesForGOpcs(OpcList: {G_BITREVERSE}, FastTypes: Standard)
1428 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32}})
1429 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32}})
1430 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {Sgpr64}})
1431 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64}});
1432
1433 addRulesForGOpcs(OpcList: {G_AMDGPU_FFBH_U32, G_AMDGPU_FFBL_B32, G_CTLZ_ZERO_POISON,
1434 G_CTTZ_ZERO_POISON})
1435 .Any(Init: {.Predicate: {UniS32, S32}, .OperandMapping: {{Sgpr32}, {Sgpr32}}})
1436 .Any(Init: {.Predicate: {DivS32, S32}, .OperandMapping: {{Vgpr32}, {Vgpr32}}})
1437 .Any(Init: {.Predicate: {UniS32, S64}, .OperandMapping: {{Sgpr32}, {Sgpr64}}})
1438 .Any(Init: {.Predicate: {DivS32, S64}, .OperandMapping: {{Vgpr32}, {Vgpr64}, SplitBitCount64To32}});
1439
1440 addRulesForGOpcs(OpcList: {G_CTPOP})
1441 .Any(Init: {.Predicate: {UniS32, S32}, .OperandMapping: {{Sgpr32}, {Sgpr32}}})
1442 .Any(Init: {.Predicate: {DivS32, S32}, .OperandMapping: {{Vgpr32}, {Vgpr32}}})
1443 .Any(Init: {.Predicate: {UniS32, S64}, .OperandMapping: {{Sgpr32}, {Sgpr64}}})
1444 .Any(Init: {.Predicate: {DivS32, S64}, .OperandMapping: {{Vgpr32}, {Vgpr64}, CtPop64To32}});
1445
1446 addRulesForGOpcs(OpcList: {G_FENCE}).Any(Init: {.Predicate: {{}}, .OperandMapping: {{}, {}}});
1447
1448 addRulesForGOpcs(OpcList: {G_READSTEADYCOUNTER, G_READCYCLECOUNTER}, FastTypes: Standard)
1449 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {}});
1450
1451 addRulesForGOpcs(OpcList: {G_GET_ROUNDING}, FastTypes: Standard)
1452 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {}, LowerGetRounding});
1453
1454 addRulesForGOpcs(OpcList: {G_SET_ROUNDING}, FastTypes: Standard)
1455 .Uni(Ty: S32, RuleApplyIDs: {{}, {SgprB32_ReadFirstLane}, LowerSetRounding})
1456 .Div(Ty: S32, RuleApplyIDs: {{}, {SgprB32_ReadFirstLane}, LowerSetRounding});
1457
1458 addRulesForGOpcs(OpcList: {G_BLOCK_ADDR}).Any(Init: {.Predicate: {UniP0}, .OperandMapping: {{SgprP0}, {}}});
1459
1460 addRulesForGOpcs(OpcList: {G_GLOBAL_VALUE})
1461 .Any(Init: {.Predicate: {UniP0}, .OperandMapping: {{SgprP0}, {}}})
1462 .Any(Init: {.Predicate: {UniP1}, .OperandMapping: {{SgprP1}, {}}})
1463 .Any(Init: {.Predicate: {UniP3}, .OperandMapping: {{SgprP3}, {}}})
1464 .Any(Init: {.Predicate: {UniP4}, .OperandMapping: {{SgprP4}, {}}})
1465 .Any(Init: {.Predicate: {UniP8}, .OperandMapping: {{SgprP8}, {}}});
1466
1467 addRulesForGOpcs(OpcList: {G_AMDGPU_WAVE_ADDRESS}).Any(Init: {.Predicate: {UniP5}, .OperandMapping: {{SgprP5}, {}}});
1468
1469 addRulesForGOpcs(OpcList: {G_AMDGPU_SPONENTRY}, FastTypes: Standard).Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {}});
1470
1471 addRulesForGOpcs(OpcList: {G_SI_CALL})
1472 .Any(Init: {.Predicate: {_, UniP0}, .OperandMapping: {{None}, {SgprP0}}})
1473 .Any(Init: {.Predicate: {_, DivP0}, .OperandMapping: {{None}, {SgprP0Call_WF}}})
1474 .Any(Init: {.Predicate: {_, UniP4}, .OperandMapping: {{None}, {SgprP4}}})
1475 .Any(Init: {.Predicate: {_, DivP4}, .OperandMapping: {{None}, {SgprP4Call_WF}}});
1476
1477 bool hasSALUFloat = ST->hasSALUFloatInsts();
1478
1479 addRulesForGOpcs(OpcList: {G_FADD, G_FMUL, G_STRICT_FADD, G_STRICT_FMUL}, FastTypes: Standard)
1480 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16, Vgpr16}}, STPred: !hasSALUFloat)
1481 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {Sgpr16, Sgpr16}}, STPred: hasSALUFloat)
1482 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
1483 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}}, STPred: hasSALUFloat)
1484 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32}}, STPred: !hasSALUFloat)
1485 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
1486 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64, Vgpr64}})
1487 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr64}})
1488 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16}}, STPred: !hasSALUFloat)
1489 .Uni(Ty: V2S16, RuleApplyIDs: {{SgprV2S16}, {SgprV2S16, SgprV2S16}, ScalarizeToS16},
1490 STPred: hasSALUFloat)
1491 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
1492 .Any(Init: {.Predicate: {UniV2S32}, .OperandMapping: {{UniInVgprV2S32}, {VgprV2S32, VgprV2S32}}})
1493 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {VgprV2S32, VgprV2S32}}})
1494 .Any(Init: {.Predicate: {UniV2S64}, .OperandMapping: {{UniInVgprV2S64}, {VgprV2S64, VgprV2S64}}})
1495 .Any(Init: {.Predicate: {DivV2S64}, .OperandMapping: {{VgprV2S64}, {VgprV2S64, VgprV2S64}}});
1496
1497 addRulesForGOpcs(OpcList: {G_FSUB, G_STRICT_FSUB}, FastTypes: Standard)
1498 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
1499 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
1500 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {Sgpr16, Sgpr16}}, STPred: hasSALUFloat)
1501 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16, Vgpr16}}, STPred: !hasSALUFloat)
1502 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}}, STPred: hasSALUFloat)
1503 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32}}, STPred: !hasSALUFloat);
1504
1505 addRulesForGOpcs(OpcList: {G_FMAD}, FastTypes: Standard)
1506 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16, Vgpr16, Vgpr16}})
1507 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16, Vgpr16}})
1508 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32, Vgpr32}})
1509 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32, Vgpr32}});
1510
1511 addRulesForGOpcs(OpcList: {G_FLDEXP, G_STRICT_FLDEXP}, FastTypes: Standard)
1512 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32}})
1513 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
1514 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16, Vgpr16}})
1515 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
1516 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64, Vgpr32}})
1517 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr32}});
1518
1519 addRulesForGOpcs(OpcList: {G_FMA, G_STRICT_FMA}, FastTypes: Standard)
1520 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16, Vgpr16}})
1521 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32, Vgpr32}})
1522 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64, Vgpr64, Vgpr64}})
1523 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr64, Vgpr64}})
1524 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16, VgprV2S16}})
1525 .Any(Init: {.Predicate: {UniV2S32}, .OperandMapping: {{UniInVgprV2S32}, {VgprV2S32, VgprV2S32, VgprV2S32}}})
1526 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {VgprV2S32, VgprV2S32, VgprV2S32}}})
1527 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {Sgpr16, Sgpr16, Sgpr16}}, STPred: hasSALUFloat)
1528 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16, Vgpr16, Vgpr16}}, STPred: !hasSALUFloat)
1529 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32, Sgpr32}}, STPred: hasSALUFloat)
1530 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32, Vgpr32}}, STPred: !hasSALUFloat)
1531 .Uni(Ty: V2S16,
1532 RuleApplyIDs: {{SgprV2S16}, {SgprV2S16, SgprV2S16, SgprV2S16}, ScalarizeToS16},
1533 STPred: hasSALUFloat)
1534 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16, VgprV2S16}},
1535 STPred: !hasSALUFloat)
1536 .Any(Init: {.Predicate: {UniV2S64}, .OperandMapping: {{UniInVgprV2S64}, {VgprV2S64, VgprV2S64, VgprV2S64}}})
1537 .Any(Init: {.Predicate: {DivV2S64}, .OperandMapping: {{VgprV2S64}, {VgprV2S64, VgprV2S64, VgprV2S64}}});
1538
1539 addRulesForGOpcs(OpcList: {G_AMDGPU_FMED3}, FastTypes: Standard)
1540 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16, Vgpr16, Vgpr16}})
1541 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16, Vgpr16}})
1542 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32, Vgpr32}})
1543 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32, Vgpr32}});
1544
1545 // TODO: This opcode is generated from the i64->i16 signed clamped pattern in
1546 // the PreLegalizerCombiner. Move the combine to RegBankCombiner to keep more
1547 // instructions on SALU.
1548 addRulesForGOpcs(OpcList: {G_AMDGPU_SMED3}, FastTypes: Standard)
1549 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32, Vgpr32}})
1550 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32, Vgpr32}});
1551
1552 // FNEG and FABS are either folded as source modifiers or can be selected as
1553 // bitwise XOR and AND with Mask. XOR and AND are available on SALU but for
1554 // targets without SALU float we still select them as VGPR since there would
1555 // be no real sgpr use.
1556 addRulesForGOpcs(OpcList: {G_FNEG, G_FABS}, FastTypes: Standard)
1557 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16}}, STPred: !hasSALUFloat)
1558 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {Sgpr16}}, STPred: hasSALUFloat)
1559 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16}})
1560 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32}}, STPred: !hasSALUFloat)
1561 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32}}, STPred: hasSALUFloat)
1562 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32}})
1563 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64}})
1564 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64}})
1565 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {VgprV2S16}}, STPred: !hasSALUFloat)
1566 .Uni(Ty: V2S16, RuleApplyIDs: {{SgprV2S16}, {SgprV2S16}, ScalarizeToS16}, STPred: hasSALUFloat)
1567 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16}})
1568 .Any(Init: {.Predicate: {UniV2S32}, .OperandMapping: {{UniInVgprV2S32}, {VgprV2S32}}})
1569 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {VgprV2S32}}});
1570
1571 addRulesForGOpcs(OpcList: {G_FCANONICALIZE}, FastTypes: Standard)
1572 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32}})
1573 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32}})
1574 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16}})
1575 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16}})
1576 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64}})
1577 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64}})
1578 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {VgprV2S16}})
1579 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16}})
1580 .Any(Init: {.Predicate: {UniV2S32}, .OperandMapping: {{UniInVgprV2S32}, {VgprV2S32}}})
1581 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {VgprV2S32}}})
1582 .Any(Init: {.Predicate: {UniV2S64}, .OperandMapping: {{UniInVgprV2S64}, {VgprV2S64}}})
1583 .Any(Init: {.Predicate: {DivV2S64}, .OperandMapping: {{VgprV2S64}, {VgprV2S64}}});
1584
1585 bool hasPST = ST->hasPseudoScalarTrans();
1586 addRulesForGOpcs(OpcList: {G_FSQRT}, FastTypes: Standard)
1587 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16}})
1588 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {Sgpr16}}, STPred: hasPST)
1589 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16}}, STPred: !hasPST);
1590
1591 addRulesForGOpcs(OpcList: {G_FPTOUI, G_FPTOSI, G_FPTOUI_SAT, G_FPTOSI_SAT})
1592 .Any(Init: {.Predicate: {UniS16, S16}, .OperandMapping: {{UniInVgprS16}, {Vgpr16}}})
1593 .Any(Init: {.Predicate: {DivS16, S16}, .OperandMapping: {{Vgpr16}, {Vgpr16}}})
1594 .Any(Init: {.Predicate: {UniS32, S16}, .OperandMapping: {{Sgpr32}, {Sgpr16}}}, STPred: hasSALUFloat)
1595 .Any(Init: {.Predicate: {UniS32, S16}, .OperandMapping: {{UniInVgprS32}, {Vgpr16}}}, STPred: !hasSALUFloat)
1596 .Any(Init: {.Predicate: {DivS32, S16}, .OperandMapping: {{Vgpr32}, {Vgpr16}}})
1597 .Any(Init: {.Predicate: {UniS16, S32}, .OperandMapping: {{Sgpr16}, {Sgpr32}}}, STPred: hasSALUFloat)
1598 .Any(Init: {.Predicate: {UniS16, S32}, .OperandMapping: {{UniInVgprS16}, {Vgpr32}}}, STPred: !hasSALUFloat)
1599 .Any(Init: {.Predicate: {DivS16, S32}, .OperandMapping: {{Vgpr16}, {Vgpr32}}})
1600 .Any(Init: {.Predicate: {UniS32, S32}, .OperandMapping: {{Sgpr32}, {Sgpr32}}}, STPred: hasSALUFloat)
1601 .Any(Init: {.Predicate: {UniS32, S32}, .OperandMapping: {{UniInVgprS32}, {Vgpr32}}}, STPred: !hasSALUFloat)
1602 .Any(Init: {.Predicate: {DivS32, S32}, .OperandMapping: {{Vgpr32}, {Vgpr32}}})
1603 .Any(Init: {.Predicate: {UniS32, S64}, .OperandMapping: {{UniInVgprS32}, {Vgpr64}}})
1604 .Any(Init: {.Predicate: {DivS32, S64}, .OperandMapping: {{Vgpr32}, {Vgpr64}}})
1605 .Any(Init: {.Predicate: {UniV2S16, V2S32}, .OperandMapping: {{UniInVgprV2S16}, {SgprV2S32}}})
1606 .Any(Init: {.Predicate: {DivV2S16, V2S32}, .OperandMapping: {{VgprV2S16}, {VgprV2S32}}});
1607
1608 addRulesForGOpcs(OpcList: {G_UITOFP, G_SITOFP})
1609 .Any(Init: {.Predicate: {UniS16, S16}, .OperandMapping: {{UniInVgprS16}, {Vgpr16}}})
1610 .Any(Init: {.Predicate: {DivS16, S16}, .OperandMapping: {{Vgpr16}, {Vgpr16}}})
1611 .Any(Init: {.Predicate: {UniS16, S32}, .OperandMapping: {{Sgpr16}, {Sgpr32}}}, STPred: hasSALUFloat)
1612 .Any(Init: {.Predicate: {UniS16, S32}, .OperandMapping: {{UniInVgprS16}, {Vgpr32}}}, STPred: !hasSALUFloat)
1613 .Any(Init: {.Predicate: {DivS16, S32}, .OperandMapping: {{Vgpr16}, {Vgpr32}}})
1614 .Any(Init: {.Predicate: {UniS32, S32}, .OperandMapping: {{Sgpr32}, {Sgpr32}}}, STPred: hasSALUFloat)
1615 .Any(Init: {.Predicate: {UniS32, S32}, .OperandMapping: {{UniInVgprS32}, {Vgpr32}}}, STPred: !hasSALUFloat)
1616 .Any(Init: {.Predicate: {DivS32, S32}, .OperandMapping: {{Vgpr32}, {Vgpr32}}})
1617 .Any(Init: {.Predicate: {UniS64, S32}, .OperandMapping: {{UniInVgprS64}, {Vgpr32}}})
1618 .Any(Init: {.Predicate: {DivS64, S32}, .OperandMapping: {{Vgpr64}, {Vgpr32}}});
1619
1620 addRulesForGOpcs(OpcList: {G_AMDGPU_S_BUFFER_PREFETCH})
1621 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {SgprV4S32_ReadFirstLane, Imm, SgprB32_ReadFirstLane}}});
1622
1623 Predicate IsDataPF([](const MachineInstr &MI) -> bool {
1624 // prefetch cache type: 0 == instruction (I$) prefetch, 1 == data prefetch.
1625 return MI.getOperand(i: 3).getImm() != 0;
1626 });
1627
1628 bool HasSMemPF = ST->hasSafeSmemPrefetch();
1629 bool HasVMemPF = ST->hasVmemPrefInsts();
1630 addRulesForGOpcs(OpcList: {G_PREFETCH})
1631 // Safe smem prefetch keeps both data and instruction prefetch.
1632 .Any(Init: {.Predicate: {UniPtr64}, .OperandMapping: {{}, {SgprPtr64}}}, STPred: HasSMemPF)
1633 // Vmem prefetch keeps data prefetch only.
1634 .Any(Init: {.Predicate: {{UniPtr64}, IsDataPF}, .OperandMapping: {{}, {SgprPtr64}}}, STPred: !HasSMemPF && HasVMemPF)
1635 .Any(Init: {.Predicate: {{UniPtr64}, IsDataPF}, .OperandMapping: {{}, {}, DeletePrefetch}},
1636 STPred: !HasSMemPF && !HasVMemPF)
1637 .Any(Init: {.Predicate: {{UniPtr64}, !IsDataPF}, .OperandMapping: {{}, {}, DeletePrefetch}}, STPred: !HasSMemPF)
1638
1639 .Any(Init: {.Predicate: {{DivPtr64}, IsDataPF}, .OperandMapping: {{}, {VgprPtr64}}}, STPred: HasVMemPF)
1640 .Any(Init: {.Predicate: {{DivPtr64}, IsDataPF}, .OperandMapping: {{}, {}, DeletePrefetch}}, STPred: !HasVMemPF)
1641 .Any(Init: {.Predicate: {{DivPtr64}, !IsDataPF}, .OperandMapping: {{}, {}, DeletePrefetch}})
1642
1643 .Any(Init: {.Predicate: {P3}, .OperandMapping: {{}, {}, DeletePrefetch}})
1644 .Any(Init: {.Predicate: {P5}, .OperandMapping: {{}, {}, DeletePrefetch}})
1645 .Any(Init: {.Predicate: {UniP6}, .OperandMapping: {{}, {SgprP6}}}, STPred: HasSMemPF)
1646 .Any(Init: {.Predicate: {UniP6}, .OperandMapping: {{}, {}, DeletePrefetch}}, STPred: !HasSMemPF);
1647
1648 addRulesForGOpcs(OpcList: {G_FPEXT})
1649 .Any(Init: {.Predicate: {DivS32, S16}, .OperandMapping: {{Vgpr32}, {Vgpr16}}})
1650 .Any(Init: {.Predicate: {UniS64, S32}, .OperandMapping: {{UniInVgprS64}, {Vgpr32}}})
1651 .Any(Init: {.Predicate: {DivS64, S32}, .OperandMapping: {{Vgpr64}, {Vgpr32}}})
1652 .Any(Init: {.Predicate: {UniS32, S16}, .OperandMapping: {{Sgpr32}, {Sgpr16}}}, STPred: hasSALUFloat)
1653 .Any(Init: {.Predicate: {UniS32, S16}, .OperandMapping: {{UniInVgprS32}, {Vgpr16}}}, STPred: !hasSALUFloat);
1654
1655 addRulesForGOpcs(OpcList: {G_AMDGPU_CVT_PK_I16_I32}, FastTypes: Standard)
1656 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {Vgpr32, Vgpr32}})
1657 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {Vgpr32, Vgpr32}});
1658
1659 addRulesForGOpcs(OpcList: {G_AMDGPU_FMIN_LEGACY, G_AMDGPU_FMAX_LEGACY}, FastTypes: Standard)
1660 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32}})
1661 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}});
1662
1663 bool hasSALUMinimumMaximumInsts = ST->hasSALUMinimumMaximumInsts();
1664
1665 addRulesForGOpcs(OpcList: {G_FMINIMUM, G_FMAXIMUM}, FastTypes: Standard)
1666 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {Sgpr16, Sgpr16}}, STPred: hasSALUMinimumMaximumInsts)
1667 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16, Vgpr16}}, STPred: !hasSALUMinimumMaximumInsts)
1668 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
1669 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}}, STPred: hasSALUMinimumMaximumInsts)
1670 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32}}, STPred: !hasSALUMinimumMaximumInsts)
1671 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
1672 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64, Vgpr64}})
1673 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr64}})
1674 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16}})
1675 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}});
1676
1677 addRulesForGOpcs(OpcList: {G_FMINNUM_IEEE, G_FMAXNUM_IEEE, G_FMINNUM, G_FMAXNUM,
1678 G_FMINIMUMNUM, G_FMAXIMUMNUM},
1679 FastTypes: Standard)
1680 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
1681 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
1682 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64, Vgpr64}})
1683 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr64}})
1684 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16}})
1685 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
1686 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {Sgpr16, Sgpr16}}, STPred: hasSALUFloat)
1687 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16, Vgpr16}}, STPred: !hasSALUFloat)
1688 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}}, STPred: hasSALUFloat)
1689 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32}}, STPred: !hasSALUFloat);
1690
1691 addRulesForGOpcs(OpcList: {G_FPTRUNC})
1692 .Any(Init: {.Predicate: {DivS16, S32}, .OperandMapping: {{Vgpr16}, {Vgpr32}}})
1693 .Any(Init: {.Predicate: {UniS32, S64}, .OperandMapping: {{UniInVgprS32}, {Vgpr64}}})
1694 .Any(Init: {.Predicate: {DivS32, S64}, .OperandMapping: {{Vgpr32}, {Vgpr64}}})
1695 .Any(Init: {.Predicate: {UniV2S16, V2S32}, .OperandMapping: {{UniInVgprV2S16}, {VgprV2S32}}})
1696 .Any(Init: {.Predicate: {DivV2S16, V2S32}, .OperandMapping: {{VgprV2S16}, {VgprV2S32}}})
1697 .Any(Init: {.Predicate: {UniS16, S32}, .OperandMapping: {{Sgpr16}, {Sgpr32}}}, STPred: hasSALUFloat)
1698 .Any(Init: {.Predicate: {UniS16, S32}, .OperandMapping: {{UniInVgprS16}, {Vgpr32}}}, STPred: !hasSALUFloat);
1699
1700 addRulesForGOpcs(OpcList: {G_INTRINSIC_FPTRUNC_ROUND})
1701 .Any(Init: {.Predicate: {UniS16, S32}, .OperandMapping: {{Sgpr16}, {Sgpr32}}}, STPred: hasSALUFloat)
1702 .Any(Init: {.Predicate: {UniS16, S32}, .OperandMapping: {{UniInVgprS16}, {Vgpr32}}}, STPred: !hasSALUFloat)
1703 .Any(Init: {.Predicate: {DivS16, S32}, .OperandMapping: {{Vgpr16}, {Vgpr32}}})
1704 .Any(Init: {.Predicate: {UniS16, S64}, .OperandMapping: {{UniInVgprS16}, {Vgpr64}}})
1705 .Any(Init: {.Predicate: {DivS16, S64}, .OperandMapping: {{Vgpr16}, {Vgpr64}}})
1706 .Any(Init: {.Predicate: {UniS32, S64}, .OperandMapping: {{UniInVgprS32}, {Vgpr64}}})
1707 .Any(Init: {.Predicate: {DivS32, S64}, .OperandMapping: {{Vgpr32}, {Vgpr64}}});
1708
1709 addRulesForGOpcs(OpcList: {G_IS_FPCLASS})
1710 .Any(Init: {.Predicate: {DivS1, S16}, .OperandMapping: {{Vcc}, {Vgpr16}}})
1711 .Any(Init: {.Predicate: {UniS1, S16}, .OperandMapping: {{UniInVcc}, {Vgpr16}}})
1712 .Any(Init: {.Predicate: {DivS1, S32}, .OperandMapping: {{Vcc}, {Vgpr32}}})
1713 .Any(Init: {.Predicate: {UniS1, S32}, .OperandMapping: {{UniInVcc}, {Vgpr32}}})
1714 .Any(Init: {.Predicate: {DivS1, S64}, .OperandMapping: {{Vcc}, {Vgpr64}}})
1715 .Any(Init: {.Predicate: {UniS1, S64}, .OperandMapping: {{UniInVcc}, {Vgpr64}}});
1716
1717 addRulesForGOpcs(OpcList: {G_FCMP}, FastTypes: Standard)
1718 .Any(Init: {.Predicate: {UniS1, _, S16}, .OperandMapping: {{Sgpr32Trunc}, {None, Sgpr16, Sgpr16}}},
1719 STPred: hasSALUFloat)
1720 .Any(Init: {.Predicate: {UniS1, _, S16}, .OperandMapping: {{UniInVcc}, {None, Vgpr16, Vgpr16}}},
1721 STPred: !hasSALUFloat)
1722 .Any(Init: {.Predicate: {DivS1, _, S16}, .OperandMapping: {{Vcc}, {None, Vgpr16, Vgpr16}}})
1723 .Any(Init: {.Predicate: {UniS1, _, S32}, .OperandMapping: {{Sgpr32Trunc}, {None, Sgpr32, Sgpr32}}},
1724 STPred: hasSALUFloat)
1725 .Any(Init: {.Predicate: {UniS1, _, S32}, .OperandMapping: {{UniInVcc}, {None, Vgpr32, Vgpr32}}},
1726 STPred: !hasSALUFloat)
1727 .Any(Init: {.Predicate: {DivS1, _, S32}, .OperandMapping: {{Vcc}, {None, Vgpr32, Vgpr32}}})
1728 .Any(Init: {.Predicate: {UniS1, _, S64}, .OperandMapping: {{UniInVcc}, {None, Vgpr64, Vgpr64}}})
1729 .Any(Init: {.Predicate: {DivS1, _, S64}, .OperandMapping: {{Vcc}, {None, Vgpr64, Vgpr64}}});
1730
1731 addRulesForGOpcs(OpcList: {G_INTRINSIC_ROUNDEVEN, G_FEXP2, G_FLOG2}, FastTypes: Standard)
1732 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16}})
1733 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16}})
1734 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32}})
1735 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32}})
1736 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64}})
1737 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64}});
1738
1739 addRulesForGOpcs(OpcList: {G_INTRINSIC_TRUNC, G_FFLOOR, G_FCEIL}, FastTypes: Standard)
1740 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16}})
1741 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {Sgpr16}}, STPred: hasSALUFloat)
1742 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16}})
1743 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32}})
1744 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32}}, STPred: hasSALUFloat)
1745 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32}})
1746 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64}})
1747 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64}});
1748
1749 addRulesForGOpcs(OpcList: {G_AMDGPU_GLOBAL_LOAD_MONITOR, G_AMDGPU_FLAT_LOAD_MONITOR},
1750 FastTypes: StandardB)
1751 .Uni(Ty: B32, RuleApplyIDs: {{UniInVgprB32}, {SgprPtr64}})
1752 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {VgprPtr64}})
1753 .Uni(Ty: B64, RuleApplyIDs: {{UniInVgprB64}, {SgprPtr64}})
1754 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {VgprPtr64}})
1755 .Uni(Ty: B128, RuleApplyIDs: {{UniInVgprB128}, {SgprPtr64}})
1756 .Div(Ty: B128, RuleApplyIDs: {{VgprB128}, {VgprPtr64}});
1757
1758 addRulesForGOpcs(OpcList: {G_AMDGPU_WHOLE_WAVE_FUNC_SETUP})
1759 .Any(Init: {.Predicate: {DivS1}, .OperandMapping: {{Vcc}, {}}});
1760
1761 addRulesForGOpcs(OpcList: {G_AMDGPU_WHOLE_WAVE_FUNC_RETURN}).Any(Init: {.Predicate: {}, .OperandMapping: {{}, {Vcc}}});
1762
1763 using namespace Intrinsic;
1764
1765 addRulesForIOpcs(OpcList: {returnaddress}).Any(Init: {.Predicate: {UniP0}, .OperandMapping: {{SgprP0}, {}}});
1766
1767 // Note: amdgcn.icmp with i1 inputs is legalized to ballot in the legalizer,
1768 // so no S1 rules are needed here.
1769 addRulesForIOpcs(OpcList: {amdgcn_icmp})
1770 .Any(Init: {.Predicate: {UniS64, _, S16}, .OperandMapping: {{Sgpr64}, {IntrId, Vgpr16, Vgpr16}}})
1771 .Any(Init: {.Predicate: {UniS64, _, S32}, .OperandMapping: {{Sgpr64}, {IntrId, Vgpr32, Vgpr32}}})
1772 .Any(Init: {.Predicate: {UniS64, _, S64}, .OperandMapping: {{Sgpr64}, {IntrId, Vgpr64, Vgpr64}}})
1773
1774 .Any(Init: {.Predicate: {UniS32, _, S16}, .OperandMapping: {{Sgpr32}, {IntrId, Vgpr16, Vgpr16}}})
1775 .Any(Init: {.Predicate: {UniS32, _, S32}, .OperandMapping: {{Sgpr32}, {IntrId, Vgpr32, Vgpr32}}})
1776 .Any(Init: {.Predicate: {UniS32, _, S64}, .OperandMapping: {{Sgpr32}, {IntrId, Vgpr64, Vgpr64}}});
1777
1778 addRulesForIOpcs(OpcList: {amdgcn_fcmp})
1779 .Any(Init: {.Predicate: {UniS64, _, S16}, .OperandMapping: {{Sgpr64}, {IntrId, Vgpr16, Vgpr16}}})
1780 .Any(Init: {.Predicate: {UniS64, _, S32}, .OperandMapping: {{Sgpr64}, {IntrId, Vgpr32, Vgpr32}}})
1781 .Any(Init: {.Predicate: {UniS64, _, S64}, .OperandMapping: {{Sgpr64}, {IntrId, Vgpr64, Vgpr64}}})
1782
1783 .Any(Init: {.Predicate: {UniS32, _, S16}, .OperandMapping: {{Sgpr32}, {IntrId, Vgpr16, Vgpr16}}})
1784 .Any(Init: {.Predicate: {UniS32, _, S32}, .OperandMapping: {{Sgpr32}, {IntrId, Vgpr32, Vgpr32}}})
1785 .Any(Init: {.Predicate: {UniS32, _, S64}, .OperandMapping: {{Sgpr32}, {IntrId, Vgpr64, Vgpr64}}});
1786
1787 addRulesForIOpcs(OpcList: {amdgcn_s_getpc}).Any(Init: {.Predicate: {UniS64, _}, .OperandMapping: {{Sgpr64}, {}}});
1788
1789 addRulesForIOpcs(OpcList: {amdgcn_s_getreg}).Any(Init: {.Predicate: {}, .OperandMapping: {{Sgpr32}, {IntrId}}});
1790
1791 addRulesForIOpcs(OpcList: {amdgcn_s_setreg})
1792 .Any(Init: {.Predicate: {_, _, S32}, .OperandMapping: {{}, {IntrId, Imm, SgprB32_ReadFirstLane}}});
1793
1794 addRulesForIOpcs(OpcList: {amdgcn_s_sendmsg, amdgcn_s_sendmsghalt})
1795 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, Imm, SgprB32_M0}}});
1796
1797 addRulesForIOpcs(OpcList: {amdgcn_s_sendmsg_rtn})
1798 .Any(Init: {.Predicate: {S32}, .OperandMapping: {{Sgpr32}, {}}})
1799 .Any(Init: {.Predicate: {S64}, .OperandMapping: {{Sgpr64}, {}}});
1800
1801 addRulesForIOpcs(OpcList: {amdgcn_s_memrealtime, amdgcn_s_memtime}, FastTypes: Standard)
1802 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {IntrId}});
1803
1804 addRulesForIOpcs(OpcList: {amdgcn_groupstaticsize, amdgcn_pops_exiting_wave_id,
1805 amdgcn_reloc_constant, amdgcn_s_get_waveid_in_workgroup},
1806 FastTypes: Standard)
1807 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {IntrId}});
1808
1809 // Intrinsics with no register operands.
1810 addRulesForIOpcs(OpcList: {amdgcn_asyncmark,
1811 amdgcn_endpgm,
1812 amdgcn_iglp_opt,
1813 amdgcn_init_exec,
1814 amdgcn_s_barrier,
1815 amdgcn_s_barrier_leave,
1816 amdgcn_s_barrier_signal,
1817 amdgcn_s_barrier_wait,
1818 amdgcn_s_monitor_sleep,
1819 amdgcn_s_nop,
1820 amdgcn_s_sethalt,
1821 amdgcn_s_setprio,
1822 amdgcn_s_setprio_inc_wg,
1823 amdgcn_s_sleep,
1824 amdgcn_s_ttracedata_imm,
1825 amdgcn_s_wait_asynccnt,
1826 amdgcn_s_wait_bvhcnt,
1827 amdgcn_s_wait_dscnt,
1828 amdgcn_s_wait_event,
1829 amdgcn_s_wait_event_export_ready,
1830 amdgcn_s_wait_expcnt,
1831 amdgcn_s_wait_kmcnt,
1832 amdgcn_s_wait_loadcnt,
1833 amdgcn_s_wait_samplecnt,
1834 amdgcn_s_wait_storecnt,
1835 amdgcn_s_wait_tensorcnt,
1836 amdgcn_s_waitcnt,
1837 amdgcn_sched_barrier,
1838 amdgcn_sched_group_barrier,
1839 amdgcn_unreachable,
1840 amdgcn_wait_asyncmark,
1841 amdgcn_wave_barrier})
1842 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {}}});
1843
1844 addRulesForIOpcs(OpcList: {amdgcn_init_exec_from_input})
1845 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, Sgpr32}}});
1846
1847 addRulesForIOpcs(OpcList: {amdgcn_s_ttracedata}).Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, SgprB32_M0}}});
1848
1849 addRulesForIOpcs(OpcList: {amdgcn_s_sleep_var})
1850 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, SgprB32_ReadFirstLane}}});
1851
1852 addRulesForIOpcs(OpcList: {amdgcn_s_barrier_join, amdgcn_s_wakeup_barrier})
1853 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, SgprB32_M0}}});
1854
1855 addRulesForIOpcs(OpcList: {amdgcn_s_barrier_signal_var, amdgcn_s_barrier_init})
1856 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, SgprB32_M0, SgprB32_M0}}});
1857
1858 addRulesForIOpcs(OpcList: {amdgcn_s_barrier_signal_isfirst})
1859 .Any(Init: {.Predicate: {UniS1}, .OperandMapping: {{Sgpr32Trunc}, {}}});
1860
1861 addRulesForIOpcs(
1862 OpcList: {amdgcn_s_get_named_barrier_state, amdgcn_s_get_barrier_state}, FastTypes: Standard)
1863 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {IntrId, SgprB32_M0}});
1864
1865 addRulesForIOpcs(OpcList: {amdgcn_flat_prefetch}).Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, VgprP0}}});
1866
1867 addRulesForIOpcs(OpcList: {amdgcn_global_prefetch}).Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, VgprP1}}});
1868
1869 addRulesForIOpcs(OpcList: {amdgcn_s_prefetch_data, amdgcn_s_prefetch_inst})
1870 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, SgprB64_ReadFirstLane, SgprB32_ReadFirstLane}}});
1871
1872 addRulesForIOpcs(OpcList: {amdgcn_class})
1873 .Any(Init: {.Predicate: {UniS1, _, S16}, .OperandMapping: {{UniInVcc}, {IntrId, Vgpr16, Vgpr32}}})
1874 .Any(Init: {.Predicate: {DivS1, _, S16}, .OperandMapping: {{Vcc}, {IntrId, Vgpr16, Vgpr32}}})
1875 .Any(Init: {.Predicate: {UniS1, _, S32}, .OperandMapping: {{UniInVcc}, {IntrId, Vgpr32, Vgpr32}}})
1876 .Any(Init: {.Predicate: {DivS1, _, S32}, .OperandMapping: {{Vcc}, {IntrId, Vgpr32, Vgpr32}}})
1877 .Any(Init: {.Predicate: {UniS1, _, S64}, .OperandMapping: {{UniInVcc}, {IntrId, Vgpr64, Vgpr32}}})
1878 .Any(Init: {.Predicate: {DivS1, _, S64}, .OperandMapping: {{Vcc}, {IntrId, Vgpr64, Vgpr32}}});
1879
1880 // This is "intrinsic lane mask" it was set to i32/i64 in llvm-ir.
1881 addRulesForIOpcs(OpcList: {amdgcn_end_cf})
1882 .Any(Init: {.Predicate: {_, UniS32}, .OperandMapping: {{}, {IntrId, Sgpr32}}})
1883 .Any(Init: {.Predicate: {_, UniS64}, .OperandMapping: {{}, {IntrId, Sgpr64}}});
1884
1885 addRulesForIOpcs(OpcList: {amdgcn_if_break}, FastTypes: Standard)
1886 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {IntrId, Vcc, Sgpr64}})
1887 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {IntrId, Vcc, Sgpr32}});
1888
1889 addRulesForIOpcs(OpcList: {amdgcn_exp})
1890 .Any(Init: {.Predicate: {_, _, _, S32, S32, S32, S32},
1891 .OperandMapping: {{}, {IntrId, Imm, Imm, Vgpr32, Vgpr32, Vgpr32, Vgpr32}}});
1892
1893 addRulesForIOpcs(OpcList: {amdgcn_exp_compr})
1894 .Any(Init: {.Predicate: {_, _, _, V2S16}, .OperandMapping: {{}, {IntrId, Imm, Imm, VgprV2S16, VgprV2S16}}});
1895
1896 addRulesForIOpcs(OpcList: {amdgcn_exp_row})
1897 .Any(Init: {.Predicate: {_, _, _, S32, S32, S32, S32, _, S32},
1898 .OperandMapping: {{},
1899 {IntrId, Imm, Imm, Vgpr32, Vgpr32, Vgpr32, Vgpr32, Imm,
1900 SgprB32_M0}}});
1901
1902 addRulesForIOpcs(OpcList: {amdgcn_lds_direct_load}, FastTypes: StandardB)
1903 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {IntrId, SgprB32_M0}});
1904
1905 addRulesForIOpcs(OpcList: {amdgcn_lds_param_load}, FastTypes: Standard)
1906 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Imm, Imm, SgprB32_M0}});
1907
1908 addRulesForIOpcs(OpcList: {amdgcn_mbcnt_lo, amdgcn_mbcnt_hi}, FastTypes: Standard)
1909 .Div(Ty: S32, RuleApplyIDs: {{}, {Vgpr32, None, Vgpr32, Vgpr32}});
1910
1911 addRulesForIOpcs(OpcList: {amdgcn_readfirstlane})
1912 .Any(Init: {.Predicate: {UniB32, _, DivB32}, .OperandMapping: {{}, {SgprB32, None, VgprB32}}})
1913 // this should not exist in the first place, it is from call lowering
1914 // readfirstlaning just in case register is not in sgpr.
1915 .Any(Init: {.Predicate: {UniS32, _, UniS32}, .OperandMapping: {{}, {Sgpr32, None, Vgpr32}}});
1916
1917 addRulesForIOpcs(OpcList: {amdgcn_readlane}, FastTypes: StandardB)
1918 .Uni(Ty: B32, RuleApplyIDs: {{SgprB32}, {IntrId, VgprB32, SgprB32_ReadFirstLane}});
1919
1920 addRulesForIOpcs(OpcList: {amdgcn_s_quadmask, amdgcn_s_wqm}, FastTypes: StandardB)
1921 .Uni(Ty: B32, RuleApplyIDs: {{SgprB32}, {IntrId, SgprB32_ReadFirstLane}})
1922 .Uni(Ty: B64, RuleApplyIDs: {{SgprB64}, {IntrId, SgprB64_ReadFirstLane}});
1923
1924 addRulesForIOpcs(OpcList: {amdgcn_writelane}, FastTypes: StandardB)
1925 .Div(Ty: B32,
1926 RuleApplyIDs: {{VgprB32},
1927 {IntrId, SgprB32_ReadFirstLane, SgprB32_ReadFirstLane, VgprB32}});
1928
1929 addRulesForIOpcs(OpcList: {amdgcn_add_max_i32, amdgcn_add_max_u32, amdgcn_add_min_i32,
1930 amdgcn_add_min_u32},
1931 FastTypes: Standard)
1932 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
1933 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
1934
1935 addRulesForIOpcs(OpcList: {amdgcn_pk_add_max_i16, amdgcn_pk_add_max_u16,
1936 amdgcn_pk_add_min_i16, amdgcn_pk_add_min_u16},
1937 FastTypes: Standard)
1938 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {IntrId, VgprV2S16, VgprV2S16, VgprV2S16}})
1939 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {IntrId, VgprV2S16, VgprV2S16, VgprV2S16}});
1940
1941 addRulesForIOpcs(OpcList: {amdgcn_permlane16, amdgcn_permlanex16}, FastTypes: Standard)
1942 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32},
1943 {IntrId, Vgpr32, Vgpr32, SgprB32_ReadFirstLane,
1944 SgprB32_ReadFirstLane}});
1945
1946 addRulesForIOpcs(OpcList: {amdgcn_permlane_bcast, amdgcn_permlane_up,
1947 amdgcn_permlane_down, amdgcn_permlane_xor},
1948 FastTypes: StandardB)
1949 .Div(Ty: B32,
1950 RuleApplyIDs: {{VgprB32},
1951 {IntrId, VgprB32, SgprB32_ReadFirstLane, SgprB32_ReadFirstLane}});
1952
1953 addRulesForIOpcs(OpcList: {amdgcn_permlane_idx_gen}, FastTypes: Standard)
1954 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, SgprB32_ReadFirstLane}});
1955
1956 addRulesForIOpcs(OpcList: {amdgcn_perm}, FastTypes: Standard)
1957 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
1958 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
1959
1960 addRulesForIOpcs(
1961 OpcList: {amdgcn_wave_reduce_add, amdgcn_wave_reduce_and, amdgcn_wave_reduce_fadd,
1962 amdgcn_wave_reduce_fmax, amdgcn_wave_reduce_fmin,
1963 amdgcn_wave_reduce_fsub, amdgcn_wave_reduce_max, amdgcn_wave_reduce_min,
1964 amdgcn_wave_reduce_or, amdgcn_wave_reduce_sub, amdgcn_wave_reduce_umax,
1965 amdgcn_wave_reduce_umin, amdgcn_wave_reduce_xor},
1966 FastTypes: Standard)
1967 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {IntrId, Sgpr32}})
1968 .Div(Ty: S32, RuleApplyIDs: {{Sgpr32ToVgprDst}, {IntrId, VgprB32}})
1969 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {IntrId, Sgpr64}})
1970 .Div(Ty: S64, RuleApplyIDs: {{Sgpr64ToVgprDst}, {IntrId, VgprB64}});
1971
1972 addRulesForIOpcs(OpcList: {amdgcn_wave_shuffle}, FastTypes: Standard)
1973 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32}})
1974 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32}});
1975
1976 addRulesForIOpcs(OpcList: {amdgcn_bitop3, amdgcn_fmad_ftz}, FastTypes: Standard)
1977 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, Vgpr16, Vgpr16, Vgpr16}})
1978 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, Vgpr16, Vgpr16, Vgpr16}})
1979 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
1980 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
1981
1982 addRulesForIOpcs(OpcList: {amdgcn_udot4, amdgcn_sdot4, amdgcn_udot8, amdgcn_sdot8,
1983 amdgcn_dot4_f32_bf8_bf8, amdgcn_dot4_f32_bf8_fp8,
1984 amdgcn_dot4_f32_fp8_fp8, amdgcn_dot4_f32_fp8_bf8},
1985 FastTypes: Standard)
1986 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
1987 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
1988
1989 addRulesForIOpcs(OpcList: {amdgcn_rsq, amdgcn_rsq_clamp}, FastTypes: Standard)
1990 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {IntrId, Sgpr16}}, STPred: hasPST)
1991 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, Vgpr16}}, STPred: !hasPST)
1992 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, Vgpr16}})
1993 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {IntrId, Sgpr32}}, STPred: hasPST)
1994 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32}}, STPred: !hasPST)
1995 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32}})
1996 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {IntrId, Vgpr64}})
1997 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {IntrId, Vgpr64}});
1998
1999 addRulesForIOpcs(OpcList: {amdgcn_mul_u24, amdgcn_mul_i24}, FastTypes: Standard)
2000 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32}})
2001 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32}})
2002 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {IntrId, Vgpr32, Vgpr32}})
2003 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {IntrId, Vgpr32, Vgpr32}});
2004
2005 addRulesForIOpcs(OpcList: {amdgcn_ds_bpermute, amdgcn_ds_bpermute_fi_b32,
2006 amdgcn_ds_permute, amdgcn_fmul_legacy, amdgcn_mulhi_i24,
2007 amdgcn_mulhi_u24},
2008 FastTypes: Standard)
2009 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32}})
2010 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32}});
2011
2012 addRulesForIOpcs(OpcList: {amdgcn_cvt_sr_bf8_f32, amdgcn_cvt_sr_fp8_f32,
2013 amdgcn_cvt_sr_fp8_f32_e5m3, amdgcn_cvt_pk_bf8_f32,
2014 amdgcn_cvt_pk_fp8_f32, amdgcn_cvt_pk_fp8_f32_e5m3},
2015 FastTypes: Standard)
2016 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
2017 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
2018
2019 addRulesForIOpcs(OpcList: {amdgcn_cvt_off_f32_i4, amdgcn_cvt_f32_bf8,
2020 amdgcn_cvt_f32_fp8, amdgcn_cvt_f32_fp8_e5m3},
2021 FastTypes: Standard)
2022 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32}})
2023 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32}});
2024
2025 addRulesForIOpcs(OpcList: {amdgcn_cvt_pk_f32_bf8, amdgcn_cvt_pk_f32_fp8})
2026 .Any(Init: {.Predicate: {UniV2S32}, .OperandMapping: {{UniInVgprV2S32}, {IntrId, Vgpr32}}})
2027 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {IntrId, Vgpr32}}});
2028
2029 addRulesForIOpcs(OpcList: {amdgcn_cvt_f16_bf8, amdgcn_cvt_f16_fp8}, FastTypes: Standard)
2030 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, Vgpr32}})
2031 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, Vgpr32}});
2032
2033 addRulesForIOpcs(OpcList: {amdgcn_cvt_pk_f16_bf8, amdgcn_cvt_pk_f16_fp8}, FastTypes: Standard)
2034 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {IntrId, Vgpr16}})
2035 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {IntrId, Vgpr16}});
2036
2037 addRulesForIOpcs(OpcList: {amdgcn_cvt_pk_bf8_f16, amdgcn_cvt_pk_fp8_f16}, FastTypes: Standard)
2038 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, VgprV2S16}})
2039 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, VgprV2S16}});
2040
2041 addRulesForIOpcs(OpcList: {amdgcn_cvt_sr_bf8_f16, amdgcn_cvt_sr_fp8_f16}, FastTypes: Standard)
2042 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr16, Vgpr32, Vgpr32}})
2043 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr16, Vgpr32, Vgpr32}});
2044
2045 addRulesForIOpcs(OpcList: {amdgcn_cvt_sr_pk_f16_f32}, FastTypes: Standard)
2046 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
2047 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
2048
2049 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_sr_fp8_f16})
2050 .Any(Init: {.Predicate: {DivS32}, .OperandMapping: {{Vgpr32}, {IntrId, Vgpr32, Vgpr16, Vgpr32, Vgpr32}}});
2051
2052 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_sr_fp8_f32})
2053 .Any(Init: {.Predicate: {DivS32}, .OperandMapping: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32, Vgpr32}}});
2054
2055 addRulesForIOpcs(OpcList: {amdgcn_cubesc, amdgcn_cubetc, amdgcn_cubema, amdgcn_cubeid,
2056 amdgcn_fma_legacy},
2057 FastTypes: Standard)
2058 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
2059 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
2060
2061 addRulesForIOpcs(OpcList: {amdgcn_frexp_mant, amdgcn_fract}, FastTypes: Standard)
2062 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, Vgpr16}})
2063 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, Vgpr16}})
2064 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32}})
2065 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32}})
2066 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {IntrId, Vgpr64}})
2067 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {IntrId, Vgpr64}});
2068
2069 addRulesForIOpcs(OpcList: {amdgcn_prng_b32})
2070 .Any(Init: {.Predicate: {UniS32}, .OperandMapping: {{UniInVgprS32}, {IntrId, Vgpr32}}})
2071 .Any(Init: {.Predicate: {DivS32}, .OperandMapping: {{Vgpr32}, {IntrId, Vgpr32}}});
2072
2073 addRulesForIOpcs(OpcList: {amdgcn_sffbh}, FastTypes: Standard)
2074 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {IntrId, Sgpr32}})
2075 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32}});
2076
2077 addRulesForIOpcs(OpcList: {amdgcn_ubfe, amdgcn_sbfe}, FastTypes: Standard)
2078 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
2079 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {IntrId, Sgpr32, Sgpr32, Sgpr32}, S_BFE})
2080 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {IntrId, Sgpr64, Sgpr32, Sgpr32}, S_BFE})
2081 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {IntrId, Vgpr64, Vgpr32, Vgpr32}, V_BFE});
2082
2083 addRulesForIOpcs(OpcList: {amdgcn_cvt_pk_i16, amdgcn_cvt_pk_u16, amdgcn_cvt_pknorm_i16,
2084 amdgcn_cvt_pknorm_u16},
2085 FastTypes: Standard)
2086 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {IntrId, Vgpr32, Vgpr32}})
2087 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {IntrId, Vgpr32, Vgpr32}});
2088
2089 addRulesForIOpcs(OpcList: {amdgcn_cvt_pkrtz}, FastTypes: Standard)
2090 .Uni(Ty: V2S16, RuleApplyIDs: {{SgprV2S16}, {IntrId, Sgpr32, Sgpr32}}, STPred: hasSALUFloat)
2091 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {IntrId, Vgpr32, Vgpr32}}, STPred: !hasSALUFloat)
2092 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {IntrId, Vgpr32, Vgpr32}});
2093
2094 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_sr_pk32_bf6_f16,
2095 amdgcn_cvt_scalef32_sr_pk32_fp6_f16,
2096 amdgcn_cvt_scalef32_sr_pk32_bf6_bf16,
2097 amdgcn_cvt_scalef32_sr_pk32_fp6_bf16},
2098 FastTypes: Standard)
2099 .Any(Init: {.Predicate: {DivV6S32}, .OperandMapping: {{VgprV6S32}, {IntrId, VgprV32S16, Vgpr32, Vgpr32}}});
2100
2101 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_sr_pk32_bf6_f32,
2102 amdgcn_cvt_scalef32_sr_pk32_fp6_f32},
2103 FastTypes: Standard)
2104 .Any(Init: {.Predicate: {DivV6S32}, .OperandMapping: {{VgprV6S32}, {IntrId, VgprV32S32, Vgpr32, Vgpr32}}});
2105
2106 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_sr_pk_fp4_f16}, FastTypes: Standard)
2107 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, VgprV2S16, Vgpr32, Vgpr32}})
2108 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, VgprV2S16, Vgpr32, Vgpr32}});
2109
2110 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_sr_pk_fp4_f32}, FastTypes: Standard)
2111 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, VgprV2S32, Vgpr32, Vgpr32}})
2112 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, VgprV2S32, Vgpr32, Vgpr32}});
2113
2114 addRulesForIOpcs(
2115 OpcList: {amdgcn_cvt_scalef32_2xpk16_fp6_f32, amdgcn_cvt_scalef32_2xpk16_bf6_f32})
2116 .Any(
2117 Init: {.Predicate: {DivV6S32}, .OperandMapping: {{VgprV6S32}, {IntrId, VgprV16S32, VgprV16S32, Vgpr32}}})
2118 .Any(Init: {.Predicate: {UniV6S32},
2119 .OperandMapping: {{UniInVgprV6S32}, {IntrId, VgprV16S32, VgprV16S32, Vgpr32}}});
2120
2121 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_f16_fp8, amdgcn_cvt_scalef32_f16_bf8},
2122 FastTypes: Standard)
2123 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {IntrId, VgprV2S16, Vgpr32, Vgpr32}})
2124 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {IntrId, VgprV2S16, Vgpr32, Vgpr32}});
2125
2126 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_f32_fp8, amdgcn_cvt_scalef32_f32_bf8},
2127 FastTypes: Standard)
2128 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32}})
2129 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32}});
2130
2131 addRulesForIOpcs(
2132 OpcList: {amdgcn_cvt_scalef32_pk16_bf6_f16, amdgcn_cvt_scalef32_pk16_fp6_f16},
2133 FastTypes: Standard)
2134 .Any(Init: {.Predicate: {DivV3S32}, .OperandMapping: {{VgprV3S32}, {IntrId, VgprV16S16, Vgpr32}}})
2135 .Any(Init: {.Predicate: {UniV3S32}, .OperandMapping: {{UniInVgprV3S32}, {IntrId, VgprV16S16, Vgpr32}}});
2136
2137 addRulesForIOpcs(
2138 OpcList: {amdgcn_cvt_scalef32_pk16_bf6_f32, amdgcn_cvt_scalef32_pk16_fp6_f32},
2139 FastTypes: Standard)
2140 .Any(Init: {.Predicate: {DivV3S32}, .OperandMapping: {{VgprV3S32}, {IntrId, VgprV16S32, Vgpr32}}})
2141 .Any(Init: {.Predicate: {UniV3S32}, .OperandMapping: {{UniInVgprV3S32}, {IntrId, VgprV16S32, Vgpr32}}});
2142
2143 addRulesForIOpcs(
2144 OpcList: {amdgcn_cvt_scalef32_pk8_bf8_f16, amdgcn_cvt_scalef32_pk8_fp8_f16},
2145 FastTypes: Standard)
2146 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {IntrId, VgprV8S16, Vgpr32}}})
2147 .Any(Init: {.Predicate: {UniV2S32}, .OperandMapping: {{UniInVgprV2S32}, {IntrId, VgprV8S16, Vgpr32}}});
2148
2149 addRulesForIOpcs(
2150 OpcList: {amdgcn_cvt_scalef32_pk8_bf8_f32, amdgcn_cvt_scalef32_pk8_fp8_f32},
2151 FastTypes: Standard)
2152 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {IntrId, VgprV8S32, Vgpr32}}})
2153 .Any(Init: {.Predicate: {UniV2S32}, .OperandMapping: {{UniInVgprV2S32}, {IntrId, VgprV8S32, Vgpr32}}});
2154
2155 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_pk8_fp4_f16}, FastTypes: Standard)
2156 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, VgprV8S16, Vgpr32}})
2157 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, VgprV8S16, Vgpr32}});
2158
2159 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_pk8_fp4_f32}, FastTypes: Standard)
2160 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, VgprV8S32, Vgpr32}})
2161 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, VgprV8S32, Vgpr32}});
2162
2163 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_sr_pk16_bf6_f16,
2164 amdgcn_cvt_scalef32_sr_pk16_fp6_f16},
2165 FastTypes: Standard)
2166 .Any(Init: {.Predicate: {DivV3S32}, .OperandMapping: {{VgprV3S32}, {IntrId, VgprV16S16, Vgpr32, Vgpr32}}})
2167 .Any(Init: {.Predicate: {UniV3S32},
2168 .OperandMapping: {{UniInVgprV3S32}, {IntrId, VgprV16S16, Vgpr32, Vgpr32}}});
2169
2170 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_sr_pk16_bf6_f32,
2171 amdgcn_cvt_scalef32_sr_pk16_fp6_f32},
2172 FastTypes: Standard)
2173 .Any(Init: {.Predicate: {DivV3S32}, .OperandMapping: {{VgprV3S32}, {IntrId, VgprV16S32, Vgpr32, Vgpr32}}})
2174 .Any(Init: {.Predicate: {UniV3S32},
2175 .OperandMapping: {{UniInVgprV3S32}, {IntrId, VgprV16S32, Vgpr32, Vgpr32}}});
2176
2177 addRulesForIOpcs(
2178 OpcList: {amdgcn_cvt_scalef32_sr_pk8_bf8_f16, amdgcn_cvt_scalef32_sr_pk8_fp8_f16},
2179 FastTypes: Standard)
2180 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {IntrId, VgprV8S16, Vgpr32, Vgpr32}}})
2181 .Any(Init: {.Predicate: {UniV2S32},
2182 .OperandMapping: {{UniInVgprV2S32}, {IntrId, VgprV8S16, Vgpr32, Vgpr32}}});
2183
2184 addRulesForIOpcs(
2185 OpcList: {amdgcn_cvt_scalef32_sr_pk8_bf8_f32, amdgcn_cvt_scalef32_sr_pk8_fp8_f32},
2186 FastTypes: Standard)
2187 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {IntrId, VgprV8S32, Vgpr32, Vgpr32}}})
2188 .Any(Init: {.Predicate: {UniV2S32},
2189 .OperandMapping: {{UniInVgprV2S32}, {IntrId, VgprV8S32, Vgpr32, Vgpr32}}});
2190
2191 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_sr_pk8_fp4_f16}, FastTypes: Standard)
2192 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, VgprV8S16, Vgpr32, Vgpr32}})
2193 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, VgprV8S16, Vgpr32, Vgpr32}});
2194
2195 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_sr_pk8_fp4_f32}, FastTypes: Standard)
2196 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, VgprV8S32, Vgpr32, Vgpr32}})
2197 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, VgprV8S32, Vgpr32, Vgpr32}});
2198
2199 addRulesForIOpcs(
2200 OpcList: {amdgcn_cvt_scale_pk16_f16_bf6, amdgcn_cvt_scale_pk16_f16_fp6}, FastTypes: Standard)
2201 .Any(Init: {.Predicate: {DivV16S16}, .OperandMapping: {{VgprV16S16}, {IntrId, VgprV3S32, Vgpr32}}})
2202 .Any(Init: {.Predicate: {UniV16S16}, .OperandMapping: {{UniInVgprV16S16}, {IntrId, VgprV3S32, Vgpr32}}});
2203
2204 addRulesForIOpcs(
2205 OpcList: {amdgcn_cvt_scale_pk16_f32_bf6, amdgcn_cvt_scale_pk16_f32_fp6}, FastTypes: Standard)
2206 .Any(Init: {.Predicate: {DivV16S32}, .OperandMapping: {{VgprV16S32}, {IntrId, VgprV3S32, Vgpr32}}})
2207 .Any(Init: {.Predicate: {UniV16S32}, .OperandMapping: {{UniInVgprV16S32}, {IntrId, VgprV3S32, Vgpr32}}});
2208
2209 addRulesForIOpcs(OpcList: {amdgcn_cvt_scale_pk8_f16_bf8, amdgcn_cvt_scale_pk8_f16_fp8},
2210 FastTypes: Standard)
2211 .Any(Init: {.Predicate: {DivV8S16}, .OperandMapping: {{VgprV8S16}, {IntrId, VgprV2S32, Vgpr32}}})
2212 .Any(Init: {.Predicate: {UniV8S16}, .OperandMapping: {{UniInVgprV8S16}, {IntrId, VgprV2S32, Vgpr32}}});
2213
2214 addRulesForIOpcs(OpcList: {amdgcn_cvt_scale_pk8_f16_fp4}, FastTypes: Standard)
2215 .Any(Init: {.Predicate: {DivV8S16}, .OperandMapping: {{VgprV8S16}, {IntrId, Vgpr32, Vgpr32}}})
2216 .Any(Init: {.Predicate: {UniV8S16}, .OperandMapping: {{UniInVgprV8S16}, {IntrId, Vgpr32, Vgpr32}}});
2217
2218 addRulesForIOpcs(OpcList: {amdgcn_cvt_scale_pk8_f32_bf8, amdgcn_cvt_scale_pk8_f32_fp8},
2219 FastTypes: Standard)
2220 .Any(Init: {.Predicate: {DivV8S32}, .OperandMapping: {{VgprV8S32}, {IntrId, VgprV2S32, Vgpr32}}})
2221 .Any(Init: {.Predicate: {UniV8S32}, .OperandMapping: {{UniInVgprV8S32}, {IntrId, VgprV2S32, Vgpr32}}});
2222
2223 addRulesForIOpcs(OpcList: {amdgcn_cvt_scale_pk8_f32_fp4}, FastTypes: Standard)
2224 .Any(Init: {.Predicate: {DivV8S32}, .OperandMapping: {{VgprV8S32}, {IntrId, Vgpr32, Vgpr32}}})
2225 .Any(Init: {.Predicate: {UniV8S32}, .OperandMapping: {{UniInVgprV8S32}, {IntrId, Vgpr32, Vgpr32}}});
2226
2227 addRulesForIOpcs(
2228 OpcList: {amdgcn_cvt_scalef32_pk32_bf6_f16, amdgcn_cvt_scalef32_pk32_fp6_f16},
2229 FastTypes: Standard)
2230 .Any(Init: {.Predicate: {DivV6S32}, .OperandMapping: {{VgprV6S32}, {IntrId, VgprV32S16, Vgpr32}}})
2231 .Any(Init: {.Predicate: {UniV6S32}, .OperandMapping: {{UniInVgprV6S32}, {IntrId, VgprV32S16, Vgpr32}}});
2232
2233 addRulesForIOpcs(
2234 OpcList: {amdgcn_cvt_scalef32_pk_fp8_f32, amdgcn_cvt_scalef32_pk_bf8_f32},
2235 FastTypes: Standard)
2236 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {IntrId, VgprV2S16, Vgpr32, Vgpr32, Vgpr32}})
2237 .Uni(Ty: V2S16,
2238 RuleApplyIDs: {{UniInVgprV2S16}, {IntrId, VgprV2S16, Vgpr32, Vgpr32, Vgpr32}});
2239
2240 addRulesForIOpcs(
2241 OpcList: {amdgcn_cvt_scalef32_pk_f32_fp8, amdgcn_cvt_scalef32_pk_f32_bf8},
2242 FastTypes: Standard)
2243 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {IntrId, Vgpr32, Vgpr32}}})
2244 .Any(Init: {.Predicate: {UniV2S32}, .OperandMapping: {{UniInVgprV2S32}, {IntrId, Vgpr32, Vgpr32}}});
2245
2246 addRulesForIOpcs(
2247 OpcList: {amdgcn_cvt_scalef32_pk_fp8_f16, amdgcn_cvt_scalef32_pk_bf8_f16},
2248 FastTypes: Standard)
2249 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {IntrId, VgprV2S16, VgprV2S16, Vgpr32}})
2250 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {IntrId, VgprV2S16, VgprV2S16, Vgpr32}});
2251
2252 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_pk_f32_fp4}, FastTypes: Standard)
2253 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {IntrId, Vgpr32, Vgpr32}}})
2254 .Any(Init: {.Predicate: {UniV2S32}, .OperandMapping: {{UniInVgprV2S32}, {IntrId, Vgpr32, Vgpr32}}});
2255
2256 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_pk_fp4_f32}, FastTypes: Standard)
2257 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32, Vgpr32}})
2258 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32, Vgpr32}});
2259
2260 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_pk_f16_fp4,
2261 amdgcn_cvt_scalef32_pk_f16_fp8,
2262 amdgcn_cvt_scalef32_pk_f16_bf8},
2263 FastTypes: Standard)
2264 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {IntrId, Vgpr32, Vgpr32}})
2265 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {IntrId, Vgpr32, Vgpr32}});
2266
2267 addRulesForIOpcs(
2268 OpcList: {amdgcn_cvt_scalef32_pk32_f32_fp6, amdgcn_cvt_scalef32_pk32_f32_bf6},
2269 FastTypes: Standard)
2270 .Any(Init: {.Predicate: {DivV32S32}, .OperandMapping: {{VgprV32S32}, {IntrId, VgprV6S32, Vgpr32}}})
2271 .Any(Init: {.Predicate: {UniV32S32}, .OperandMapping: {{UniInVgprV32S32}, {IntrId, VgprV6S32, Vgpr32}}});
2272
2273 addRulesForIOpcs(
2274 OpcList: {amdgcn_cvt_scalef32_pk32_f16_fp6, amdgcn_cvt_scalef32_pk32_f16_bf6},
2275 FastTypes: Standard)
2276 .Any(Init: {.Predicate: {DivV32S16}, .OperandMapping: {{VgprV32S16}, {IntrId, VgprV6S32, Vgpr32}}})
2277 .Any(Init: {.Predicate: {UniV32S16}, .OperandMapping: {{UniInVgprV32S16}, {IntrId, VgprV6S32, Vgpr32}}});
2278
2279 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_pk_fp4_f16}, FastTypes: Standard)
2280 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, VgprV2S16, Vgpr32}})
2281 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, VgprV2S16, Vgpr32}});
2282
2283 addRulesForIOpcs(OpcList: {amdgcn_global_load_tr_b64})
2284 .Any(Init: {.Predicate: {DivB64, _, UniP1}, .OperandMapping: {{VgprB64}, {IntrId, SgprP1}}})
2285 .Any(Init: {.Predicate: {DivB64, _, DivP1}, .OperandMapping: {{VgprB64}, {IntrId, VgprP1}}})
2286 .Any(Init: {.Predicate: {DivB32, _, UniP1}, .OperandMapping: {{VgprB32}, {IntrId, SgprP1}}})
2287 .Any(Init: {.Predicate: {DivB32, _, DivP1}, .OperandMapping: {{VgprB32}, {IntrId, VgprP1}}});
2288
2289 addRulesForIOpcs(OpcList: {amdgcn_global_load_tr_b128})
2290 .Any(Init: {.Predicate: {DivB64, _, UniP1}, .OperandMapping: {{VgprB64}, {IntrId, SgprP1}}})
2291 .Any(Init: {.Predicate: {DivB64, _, DivP1}, .OperandMapping: {{VgprB64}, {IntrId, VgprP1}}})
2292 .Any(Init: {.Predicate: {DivB128, _, UniP1}, .OperandMapping: {{VgprB128}, {IntrId, SgprP1}}})
2293 .Any(Init: {.Predicate: {DivB128, _, DivP1}, .OperandMapping: {{VgprB128}, {IntrId, VgprP1}}});
2294
2295 addRulesForIOpcs(OpcList: {amdgcn_global_load_tr4_b64})
2296 .Any(Init: {.Predicate: {DivV2S32, _, UniP1}, .OperandMapping: {{VgprV2S32}, {IntrId, SgprP1}}})
2297 .Any(Init: {.Predicate: {DivV2S32, _, DivP1}, .OperandMapping: {{VgprV2S32}, {IntrId, VgprP1}}});
2298
2299 addRulesForIOpcs(OpcList: {amdgcn_global_load_tr6_b96})
2300 .Any(Init: {.Predicate: {DivV3S32, _, UniP1}, .OperandMapping: {{VgprV3S32}, {IntrId, SgprP1}}})
2301 .Any(Init: {.Predicate: {DivV3S32, _, DivP1}, .OperandMapping: {{VgprV3S32}, {IntrId, VgprP1}}});
2302
2303 addRulesForIOpcs(OpcList: {amdgcn_ds_load_tr4_b64, amdgcn_ds_load_tr8_b64})
2304 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {IntrId, VgprP3}}});
2305
2306 addRulesForIOpcs(OpcList: {amdgcn_ds_load_tr6_b96})
2307 .Any(Init: {.Predicate: {DivV3S32}, .OperandMapping: {{VgprV3S32}, {IntrId, VgprP3}}});
2308
2309 addRulesForIOpcs(OpcList: {amdgcn_ds_load_tr16_b128})
2310 .Any(Init: {.Predicate: {DivB128}, .OperandMapping: {{VgprB128}, {IntrId, VgprP3}}});
2311
2312 addRulesForIOpcs(OpcList: {amdgcn_global_atomic_ordered_add_b64})
2313 .Any(Init: {.Predicate: {DivS64}, .OperandMapping: {{Vgpr64}, {IntrId, VgprP1, Vgpr64}}});
2314
2315 addRulesForIOpcs(
2316 OpcList: {amdgcn_global_atomic_fmin_num, amdgcn_global_atomic_fmax_num}, FastTypes: Standard)
2317 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, VgprP1, Vgpr32}});
2318
2319 addRulesForIOpcs(OpcList: {amdgcn_flat_atomic_fmin_num, amdgcn_flat_atomic_fmax_num},
2320 FastTypes: Standard)
2321 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, VgprP0, Vgpr32}});
2322
2323 addRulesForIOpcs(OpcList: {amdgcn_raw_buffer_load_lds})
2324 .Any(Init: {.Predicate: {_}, .OperandMapping: {{}, {IntrId, SgprV4S32, SgprP3, Imm, Vgpr32, Sgpr32}}});
2325
2326 addRulesForIOpcs(OpcList: {amdgcn_raw_buffer_load_async_lds})
2327 .Any(Init: {.Predicate: {_}, .OperandMapping: {{}, {IntrId, SgprV4S32, SgprB32_M0, Imm, Vgpr32, Sgpr32}}});
2328
2329 addRulesForIOpcs(OpcList: {amdgcn_struct_buffer_load_async_lds})
2330 .Any(
2331 Init: {.Predicate: {_},
2332 .OperandMapping: {{}, {IntrId, SgprV4S32, SgprB32_M0, Imm, Vgpr32, Vgpr32, Sgpr32}}});
2333
2334 addRulesForIOpcs(OpcList: {amdgcn_struct_buffer_load_lds})
2335 .Any(Init: {.Predicate: {_},
2336 .OperandMapping: {{}, {IntrId, SgprV4S32, SgprP3, Imm, Vgpr32, Vgpr32, Sgpr32}}});
2337
2338 addRulesForIOpcs(OpcList: {amdgcn_raw_ptr_buffer_load_lds})
2339 .Any(Init: {.Predicate: {_}, .OperandMapping: {{}, {IntrId, SgprP8, SgprP3, Imm, Vgpr32, Sgpr32}}});
2340
2341 addRulesForIOpcs(OpcList: {amdgcn_raw_ptr_buffer_load_async_lds})
2342 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, SgprP8, SgprB32_M0, Imm, VgprB32, SgprB32}}});
2343
2344 addRulesForIOpcs(OpcList: {amdgcn_struct_ptr_buffer_load_async_lds})
2345 .Any(Init: {.Predicate: {_},
2346 .OperandMapping: {{}, {IntrId, SgprP8, SgprB32_M0, Imm, Vgpr32, Vgpr32, Sgpr32}}});
2347
2348 addRulesForIOpcs(OpcList: {amdgcn_struct_ptr_buffer_load_lds})
2349 .Any(Init: {.Predicate: {_}, .OperandMapping: {{}, {IntrId, SgprP8, SgprP3, Imm, Vgpr32, Vgpr32, Sgpr32}}});
2350
2351 addRulesForIOpcs(
2352 OpcList: {amdgcn_global_load_lds, amdgcn_load_to_lds, amdgcn_load_async_to_lds})
2353 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, VgprP1, SgprB32_M0}}});
2354
2355 addRulesForIOpcs(OpcList: {amdgcn_global_load_async_to_lds_b8,
2356 amdgcn_global_load_async_to_lds_b32,
2357 amdgcn_global_load_async_to_lds_b64,
2358 amdgcn_global_load_async_to_lds_b128,
2359 amdgcn_global_store_async_from_lds_b8,
2360 amdgcn_global_store_async_from_lds_b32,
2361 amdgcn_global_store_async_from_lds_b64,
2362 amdgcn_global_store_async_from_lds_b128})
2363 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, VgprP1, VgprP3}}});
2364
2365 addRulesForIOpcs(OpcList: {amdgcn_global_load_async_lds})
2366 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, VgprP1, SgprB32_M0}}});
2367
2368 addRulesForIOpcs(OpcList: {amdgcn_tensor_load_to_lds, amdgcn_tensor_store_from_lds})
2369 .Any(Init: {.Predicate: {},
2370 .OperandMapping: {{},
2371 {IntrId, SgprV4S32_ReadFirstLane, SgprV8S32_ReadFirstLane,
2372 SgprV4S32_ReadFirstLane, SgprV4S32_ReadFirstLane,
2373 SgprV8S32_ReadFirstLane}}});
2374
2375 addRulesForIOpcs(OpcList: {amdgcn_cluster_load_b32})
2376 .Any(Init: {.Predicate: {UniB32}, .OperandMapping: {{UniInVgprB32}, {IntrId, SgprP1, Imm, SgprB32_M0}}})
2377 .Any(Init: {.Predicate: {DivB32, _, UniP1}, .OperandMapping: {{VgprB32}, {IntrId, SgprP1, Imm, SgprB32_M0}}})
2378 .Any(
2379 Init: {.Predicate: {DivB32, _, DivP1}, .OperandMapping: {{VgprB32}, {IntrId, VgprP1, Imm, SgprB32_M0}}});
2380
2381 addRulesForIOpcs(OpcList: {amdgcn_cluster_load_b64})
2382 .Any(Init: {.Predicate: {UniB64}, .OperandMapping: {{UniInVgprB64}, {IntrId, SgprP1, Imm, SgprB32_M0}}})
2383 .Any(Init: {.Predicate: {DivB64, _, UniP1}, .OperandMapping: {{VgprB64}, {IntrId, SgprP1, Imm, SgprB32_M0}}})
2384 .Any(
2385 Init: {.Predicate: {DivB64, _, DivP1}, .OperandMapping: {{VgprB64}, {IntrId, VgprP1, Imm, SgprB32_M0}}});
2386
2387 addRulesForIOpcs(OpcList: {amdgcn_cluster_load_b128})
2388 .Any(Init: {.Predicate: {UniB128}, .OperandMapping: {{UniInVgprB128}, {IntrId, SgprP1, Imm, SgprB32_M0}}})
2389 .Any(Init: {.Predicate: {DivB128, _, UniP1},
2390 .OperandMapping: {{VgprB128}, {IntrId, SgprP1, Imm, SgprB32_M0}}})
2391 .Any(Init: {.Predicate: {DivB128, _, DivP1},
2392 .OperandMapping: {{VgprB128}, {IntrId, VgprP1, Imm, SgprB32_M0}}});
2393
2394 addRulesForIOpcs(OpcList: {amdgcn_cluster_load_async_to_lds_b8,
2395 amdgcn_cluster_load_async_to_lds_b32,
2396 amdgcn_cluster_load_async_to_lds_b64,
2397 amdgcn_cluster_load_async_to_lds_b128})
2398 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, VgprP1, VgprP3, Imm, Imm, SgprB32_M0}}});
2399
2400 addRulesForIOpcs(OpcList: {amdgcn_perm_pk16_b4_u4}, FastTypes: StandardB)
2401 .Uni(Ty: B64, RuleApplyIDs: {{UniInVgprB64}, {IntrId, Vgpr32, Vgpr32, VgprV2S32}})
2402 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {IntrId, Vgpr32, Vgpr32, VgprV2S32}});
2403
2404 addRulesForIOpcs(OpcList: {amdgcn_perm_pk16_b6_u4}, FastTypes: StandardB)
2405 .Uni(Ty: B96, RuleApplyIDs: {{UniInVgprB96}, {IntrId, Vgpr32, VgprB64, VgprV2S32}})
2406 .Div(Ty: B96, RuleApplyIDs: {{VgprB96}, {IntrId, Vgpr32, VgprB64, VgprV2S32}});
2407
2408 addRulesForIOpcs(OpcList: {amdgcn_perm_pk16_b8_u4}, FastTypes: StandardB)
2409 .Uni(Ty: B128, RuleApplyIDs: {{UniInVgprB128}, {IntrId, VgprB64, VgprB64, VgprV2S32}})
2410 .Div(Ty: B128, RuleApplyIDs: {{VgprB128}, {IntrId, VgprB64, VgprB64, VgprV2S32}});
2411
2412 addRulesForIOpcs(OpcList: {amdgcn_wwm, amdgcn_strict_wwm, amdgcn_wqm, amdgcn_softwqm,
2413 amdgcn_strict_wqm},
2414 FastTypes: StandardB)
2415 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {IntrId, VgprB32}})
2416 .Uni(Ty: B32, RuleApplyIDs: {{SgprB32}, {IntrId, SgprB32}})
2417 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {IntrId, VgprB64}})
2418 .Uni(Ty: B64, RuleApplyIDs: {{SgprB64}, {IntrId, SgprB64}})
2419 .Div(Ty: B96, RuleApplyIDs: {{VgprB96}, {IntrId, VgprB96}})
2420 .Uni(Ty: B96, RuleApplyIDs: {{SgprB96}, {IntrId, SgprB96}})
2421 .Div(Ty: B128, RuleApplyIDs: {{VgprB128}, {IntrId, VgprB128}})
2422 .Uni(Ty: B128, RuleApplyIDs: {{SgprB128}, {IntrId, SgprB128}})
2423 .Any(Init: {.Predicate: {UniB256}, .OperandMapping: {{SgprB256}, {IntrId, SgprB256}}})
2424 .Any(Init: {.Predicate: {DivB256}, .OperandMapping: {{VgprB256}, {IntrId, VgprB256}}})
2425 .Any(Init: {.Predicate: {UniB512}, .OperandMapping: {{SgprB512}, {IntrId, SgprB512}}})
2426 .Any(Init: {.Predicate: {DivB512}, .OperandMapping: {{VgprB512}, {IntrId, VgprB512}}});
2427
2428 addRulesForIOpcs(OpcList: {amdgcn_init_whole_wave}).Any(Init: {.Predicate: {DivS1}, .OperandMapping: {{Vcc}, {IntrId}}});
2429
2430 addRulesForIOpcs(OpcList: {amdgcn_kill, amdgcn_wqm_demote})
2431 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, Vcc}}});
2432
2433 addRulesForIOpcs(OpcList: {amdgcn_set_inactive}, FastTypes: StandardB)
2434 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {IntrId, VgprB32, VgprB32}});
2435
2436 addRulesForIOpcs(OpcList: {amdgcn_set_inactive_chain_arg}, FastTypes: Standard)
2437 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32}});
2438
2439 addRulesForIOpcs(OpcList: {amdgcn_cvt_sr_bf16_f32, amdgcn_cvt_sr_f16_f32}, FastTypes: Standard)
2440 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {IntrId, VgprV2S16, Vgpr32, Vgpr32}});
2441
2442 addRulesForIOpcs(OpcList: {amdgcn_ballot}, FastTypes: Standard)
2443 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {IntrId, Vcc}})
2444 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {IntrId, Vcc}});
2445
2446 addRulesForIOpcs(OpcList: {amdgcn_inverse_ballot})
2447 .Any(Init: {.Predicate: {DivS1, _, S32}, .OperandMapping: {{Vcc}, {IntrId, SgprB32_ReadFirstLane}}})
2448 .Any(Init: {.Predicate: {DivS1, _, S64}, .OperandMapping: {{Vcc}, {IntrId, SgprB64_ReadFirstLane}}});
2449
2450 addRulesForIOpcs(OpcList: {amdgcn_live_mask, amdgcn_ps_live})
2451 .Any(Init: {.Predicate: {DivS1}, .OperandMapping: {{Vcc}, {}}});
2452
2453 addRulesForIOpcs(OpcList: {amdgcn_mov_dpp, amdgcn_mov_dpp8}, FastTypes: StandardB)
2454 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {IntrId, VgprB32}})
2455 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {IntrId, VgprB64}});
2456
2457 addRulesForIOpcs(OpcList: {amdgcn_update_dpp}, FastTypes: StandardB)
2458 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {IntrId, VgprB32, VgprB32}})
2459 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {IntrId, VgprB64, VgprB64}});
2460
2461 addRulesForIOpcs(OpcList: {amdgcn_sin, amdgcn_cos}, FastTypes: Standard)
2462 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, Vgpr16}})
2463 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, Vgpr16}})
2464 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32}})
2465 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32}});
2466
2467 addRulesForIOpcs(OpcList: {amdgcn_trig_preop}, FastTypes: Standard)
2468 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {IntrId, Vgpr64, Vgpr32}})
2469 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {IntrId, Vgpr64, Vgpr32}});
2470
2471 addRulesForIOpcs(OpcList: {amdgcn_exp2}, FastTypes: Standard)
2472 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, Vgpr16}})
2473 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {IntrId, Sgpr16}}, STPred: hasPST)
2474 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, Vgpr16}}, STPred: !hasPST)
2475 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32}})
2476 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {IntrId, Sgpr32}}, STPred: hasPST)
2477 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32}}, STPred: !hasPST);
2478
2479 addRulesForIOpcs(OpcList: {amdgcn_rcp, amdgcn_sqrt}, FastTypes: Standard)
2480 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, Vgpr16}})
2481 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {IntrId, Sgpr16}}, STPred: hasPST)
2482 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, Vgpr16}}, STPred: !hasPST)
2483 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32}})
2484 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {IntrId, Sgpr32}}, STPred: hasPST)
2485 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32}}, STPred: !hasPST)
2486 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {IntrId, Vgpr64}})
2487 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {IntrId, Vgpr64}});
2488
2489 addRulesForIOpcs(OpcList: {amdgcn_log}, FastTypes: Standard)
2490 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, Vgpr16}})
2491 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {IntrId, Sgpr16}}, STPred: hasPST)
2492 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, Vgpr16}}, STPred: !hasPST)
2493 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32}})
2494 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {IntrId, Sgpr32}}, STPred: hasPST)
2495 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32}}, STPred: !hasPST);
2496
2497 addRulesForIOpcs(OpcList: {amdgcn_ds_atomic_async_barrier_arrive_b64})
2498 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, VgprP3}}});
2499
2500 addRulesForIOpcs(OpcList: {amdgcn_ds_atomic_barrier_arrive_rtn_b64}, FastTypes: Standard)
2501 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {IntrId, VgprP3, Vgpr64}});
2502
2503 addRulesForIOpcs(OpcList: {amdgcn_ds_add_gs_reg_rtn, amdgcn_ds_sub_gs_reg_rtn},
2504 FastTypes: Standard)
2505 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32}})
2506 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {IntrId, Vgpr32}});
2507
2508 addRulesForIOpcs(OpcList: {amdgcn_ds_append, amdgcn_ds_consume}, FastTypes: Standard)
2509 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, SgprB32_M0}})
2510 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, SgprB32_M0}});
2511
2512 addRulesForIOpcs(
2513 OpcList: {amdgcn_ds_bvh_stack_rtn, amdgcn_ds_bvh_stack_push4_pop1_rtn}, FastTypes: Standard)
2514 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32, Vgpr32}, {IntrId, Vgpr32, Vgpr32, VgprV4S32}});
2515
2516 addRulesForIOpcs(OpcList: {amdgcn_ds_bvh_stack_push8_pop1_rtn}, FastTypes: Standard)
2517 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32, Vgpr32}, {IntrId, Vgpr32, Vgpr32, VgprV8S32}});
2518
2519 addRulesForIOpcs(OpcList: {amdgcn_ds_bvh_stack_push8_pop2_rtn}, FastTypes: Standard)
2520 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64, Vgpr32}, {IntrId, Vgpr32, Vgpr32, VgprV8S32}});
2521
2522 addRulesForIOpcs(OpcList: {amdgcn_ds_gws_sema_p, amdgcn_ds_gws_sema_v,
2523 amdgcn_ds_gws_sema_release_all})
2524 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, SgprB32_M0}}});
2525
2526 addRulesForIOpcs(
2527 OpcList: {amdgcn_ds_gws_barrier, amdgcn_ds_gws_init, amdgcn_ds_gws_sema_br})
2528 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, Vgpr32, SgprB32_M0}}});
2529
2530 addRulesForIOpcs(OpcList: {amdgcn_ds_ordered_add, amdgcn_ds_ordered_swap}, FastTypes: Standard)
2531 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, SgprB32_M0, Vgpr32}});
2532
2533 addRulesForIOpcs(OpcList: {amdgcn_ds_swizzle}, FastTypes: Standard)
2534 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32}})
2535 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32}});
2536
2537 addRulesForIOpcs(OpcList: {amdgcn_permlane16_var, amdgcn_permlanex16_var}, FastTypes: Standard)
2538 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
2539
2540 addRulesForIOpcs(OpcList: {amdgcn_permlane16_swap, amdgcn_permlane32_swap}, FastTypes: Standard)
2541 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32, Vgpr32}, {IntrId, Vgpr32, Vgpr32}});
2542
2543 addRulesForIOpcs(OpcList: {amdgcn_permlane64}, FastTypes: StandardB)
2544 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {IntrId, VgprB32}});
2545
2546 addRulesForIOpcs(OpcList: {amdgcn_ds_read_tr4_b64, amdgcn_ds_read_tr8_b64})
2547 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {IntrId, VgprP3}}});
2548
2549 addRulesForIOpcs(OpcList: {amdgcn_ds_read_tr6_b96})
2550 .Any(Init: {.Predicate: {DivV3S32}, .OperandMapping: {{VgprV3S32}, {IntrId, VgprP3}}});
2551
2552 addRulesForIOpcs(OpcList: {amdgcn_ds_read_tr16_b64})
2553 .Any(Init: {.Predicate: {DivV4S16}, .OperandMapping: {{VgprV4S16}, {IntrId, VgprP3}}});
2554
2555 addRulesForIOpcs(OpcList: {amdgcn_interp_p1}, FastTypes: Standard)
2556 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Imm, Imm, SgprB32_M0}});
2557
2558 addRulesForIOpcs(OpcList: {amdgcn_interp_p1_f16}, FastTypes: Standard)
2559 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Imm, Imm, Imm, SgprB32_M0}});
2560
2561 addRulesForIOpcs(OpcList: {amdgcn_interp_p2}, FastTypes: Standard)
2562 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Imm, Imm, SgprB32_M0}});
2563
2564 addRulesForIOpcs(OpcList: {amdgcn_interp_p2_f16}, FastTypes: Standard)
2565 .Div(Ty: S16,
2566 RuleApplyIDs: {{Vgpr16}, {IntrId, Vgpr32, Vgpr32, Imm, Imm, Imm, SgprB32_M0}});
2567
2568 addRulesForIOpcs(OpcList: {amdgcn_interp_mov}, FastTypes: Standard)
2569 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Imm, Imm, Imm, SgprB32_M0}});
2570
2571 addRulesForIOpcs(OpcList: {amdgcn_interp_inreg_p10, amdgcn_interp_inreg_p2,
2572 amdgcn_interp_inreg_p10_f16, amdgcn_interp_p10_rtz_f16},
2573 FastTypes: Standard)
2574 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
2575 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
2576
2577 addRulesForIOpcs(OpcList: {amdgcn_interp_inreg_p2_f16, amdgcn_interp_p2_rtz_f16},
2578 FastTypes: Standard)
2579 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
2580 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
2581
2582 addRulesForIOpcs(OpcList: {amdgcn_frexp_exp})
2583 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{UniInVgprS16}, {IntrId, Vgpr16}}})
2584 .Any(Init: {.Predicate: {DivS16}, .OperandMapping: {{Vgpr16}, {IntrId, Vgpr16}}})
2585 .Any(Init: {.Predicate: {UniS32, _, S32}, .OperandMapping: {{UniInVgprS32}, {IntrId, Vgpr32}}})
2586 .Any(Init: {.Predicate: {DivS32, _, S32}, .OperandMapping: {{Vgpr32}, {IntrId, Vgpr32}}})
2587 .Any(Init: {.Predicate: {UniS32, _, S64}, .OperandMapping: {{UniInVgprS32}, {IntrId, Vgpr64}}})
2588 .Any(Init: {.Predicate: {DivS32, _, S64}, .OperandMapping: {{Vgpr32}, {IntrId, Vgpr64}}});
2589
2590 addRulesForIOpcs(OpcList: {amdgcn_div_fmas}, FastTypes: Standard)
2591 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32, Vcc}})
2592 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32, Vcc}})
2593 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {IntrId, Vgpr64, Vgpr64, Vgpr64, Vcc}})
2594 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {IntrId, Vgpr64, Vgpr64, Vgpr64, Vcc}});
2595
2596 addRulesForIOpcs(OpcList: {amdgcn_div_fixup}, FastTypes: Standard)
2597 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, Vgpr16, Vgpr16, Vgpr16}})
2598 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, Vgpr16, Vgpr16, Vgpr16}})
2599 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
2600 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
2601 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {IntrId, Vgpr64, Vgpr64, Vgpr64}})
2602 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {IntrId, Vgpr64, Vgpr64, Vgpr64}});
2603
2604 addRulesForIOpcs(OpcList: {amdgcn_div_scale}, FastTypes: Standard)
2605 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32, Vcc}, {IntrId, Vgpr32, Vgpr32}})
2606 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32, UniInVcc}, {IntrId, Vgpr32, Vgpr32}})
2607 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64, Vcc}, {IntrId, Vgpr64, Vgpr64}})
2608 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64, UniInVcc}, {IntrId, Vgpr64, Vgpr64}});
2609
2610 addRulesForIOpcs(OpcList: {amdgcn_fdot2, amdgcn_sdot2, amdgcn_udot2}, FastTypes: Standard)
2611 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, VgprV2S16, VgprV2S16, Vgpr32}})
2612 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, VgprV2S16, VgprV2S16, Vgpr32}});
2613
2614 addRulesForIOpcs(OpcList: {amdgcn_fdot2_f16_f16}, FastTypes: Standard)
2615 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, VgprV2S16, VgprV2S16, Vgpr16}})
2616 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, VgprV2S16, VgprV2S16, Vgpr16}});
2617
2618 addRulesForIOpcs(OpcList: {amdgcn_sudot4, amdgcn_sudot8}, FastTypes: Standard)
2619 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Imm, Vgpr32, Imm, Vgpr32, Vgpr32}})
2620 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Imm, Vgpr32, Imm, Vgpr32, Vgpr32}});
2621
2622 addRulesForIOpcs(OpcList: {amdgcn_s_alloc_vgpr})
2623 .Any(Init: {.Predicate: {UniS1}, .OperandMapping: {{Sgpr32Trunc}, {IntrId, SgprB32_ReadFirstLane}}});
2624
2625 addRulesForIOpcs(OpcList: {amdgcn_sat_pk4_i4_i8, amdgcn_sat_pk4_u4_u8}, FastTypes: Standard)
2626 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, Vgpr32}})
2627 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, Vgpr32}});
2628
2629 bool HasGFX90AInsts = ST->hasGFX90AInsts();
2630
2631 // On gfx90a+ both AGPR-form and VGPR-form exists
2632 addRulesForIOpcs(OpcList: {amdgcn_mfma_f32_32x32x1f32, amdgcn_mfma_f32_16x16x1f32,
2633 amdgcn_mfma_f32_4x4x1f32, amdgcn_mfma_f32_32x32x2f32,
2634 amdgcn_mfma_f32_16x16x4f32, amdgcn_mfma_f32_32x32x4f16,
2635 amdgcn_mfma_f32_16x16x4f16, amdgcn_mfma_f32_4x4x4f16,
2636 amdgcn_mfma_f32_32x32x8f16, amdgcn_mfma_f32_16x16x16f16,
2637 amdgcn_mfma_i32_32x32x4i8, amdgcn_mfma_i32_16x16x4i8,
2638 amdgcn_mfma_i32_4x4x4i8, amdgcn_mfma_i32_32x32x8i8,
2639 amdgcn_mfma_i32_16x16x16i8, amdgcn_mfma_f32_32x32x2bf16,
2640 amdgcn_mfma_f32_16x16x2bf16, amdgcn_mfma_f32_4x4x2bf16,
2641 amdgcn_mfma_f32_32x32x4bf16, amdgcn_mfma_f32_16x16x8bf16})
2642 .Any(Init: {.Predicate: {DivAnyTy},
2643 .OperandMapping: {{AgprAnyTy}, {IntrId, VgprAnyTy, VgprAnyTy, AgprAnyTy}}},
2644 STPred: !HasGFX90AInsts)
2645 .Any(Init: {.Predicate: {DivAnyTy},
2646 .OperandMapping: {{VgprOrAgprAnyTy},
2647 {IntrId, VgprAnyTy, VgprAnyTy, VgprOrAgprAnyTy}}},
2648 STPred: HasGFX90AInsts);
2649
2650 // gfx90a+ only MFMAs
2651 addRulesForIOpcs(
2652 OpcList: {
2653 amdgcn_mfma_f32_32x32x4bf16_1k,
2654 amdgcn_mfma_f32_16x16x4bf16_1k,
2655 amdgcn_mfma_f32_4x4x4bf16_1k,
2656 amdgcn_mfma_f32_32x32x8bf16_1k,
2657 amdgcn_mfma_f32_16x16x16bf16_1k,
2658 amdgcn_mfma_f64_16x16x4f64,
2659 amdgcn_mfma_f64_4x4x4f64,
2660 amdgcn_mfma_i32_16x16x32_i8,
2661 amdgcn_mfma_i32_32x32x16_i8,
2662 amdgcn_mfma_f32_16x16x8_xf32,
2663 amdgcn_mfma_f32_32x32x4_xf32,
2664 amdgcn_mfma_f32_16x16x32_bf8_bf8,
2665 amdgcn_mfma_f32_16x16x32_bf8_fp8,
2666 amdgcn_mfma_f32_16x16x32_fp8_bf8,
2667 amdgcn_mfma_f32_16x16x32_fp8_fp8,
2668 amdgcn_mfma_f32_32x32x16_bf8_bf8,
2669 amdgcn_mfma_f32_32x32x16_bf8_fp8,
2670 amdgcn_mfma_f32_32x32x16_fp8_bf8,
2671 amdgcn_mfma_f32_32x32x16_fp8_fp8,
2672 // gfx950
2673 amdgcn_mfma_f32_16x16x32_f16,
2674 amdgcn_mfma_f32_32x32x16_f16,
2675 amdgcn_mfma_i32_16x16x64_i8,
2676 amdgcn_mfma_i32_32x32x32_i8,
2677 // TODO: bf16 variants fail in IRTranslator.
2678 // amdgcn_mfma_f32_16x16x32_bf16, amdgcn_mfma_f32_32x32x16_bf16,
2679 })
2680 .Any(Init: {.Predicate: {DivAnyTy},
2681 .OperandMapping: {{VgprOrAgprAnyTy},
2682 {IntrId, VgprAnyTy, VgprAnyTy, VgprOrAgprAnyTy}}});
2683
2684 addRulesForIOpcs(
2685 OpcList: {// gfx942+
2686 amdgcn_smfmac_f32_16x16x32_f16, amdgcn_smfmac_f32_32x32x16_f16,
2687 amdgcn_smfmac_f32_16x16x32_bf16, amdgcn_smfmac_f32_32x32x16_bf16,
2688 amdgcn_smfmac_i32_16x16x64_i8, amdgcn_smfmac_i32_32x32x32_i8,
2689 amdgcn_smfmac_f32_16x16x64_bf8_bf8, amdgcn_smfmac_f32_16x16x64_bf8_fp8,
2690 amdgcn_smfmac_f32_16x16x64_fp8_bf8, amdgcn_smfmac_f32_16x16x64_fp8_fp8,
2691 amdgcn_smfmac_f32_32x32x32_bf8_bf8, amdgcn_smfmac_f32_32x32x32_bf8_fp8,
2692 amdgcn_smfmac_f32_32x32x32_fp8_bf8, amdgcn_smfmac_f32_32x32x32_fp8_fp8,
2693 // gfx950+
2694 amdgcn_smfmac_f32_16x16x64_f16, amdgcn_smfmac_f32_32x32x32_f16,
2695 amdgcn_smfmac_i32_16x16x128_i8, amdgcn_smfmac_i32_32x32x64_i8,
2696 amdgcn_smfmac_f32_16x16x128_bf8_bf8, amdgcn_smfmac_f32_16x16x128_bf8_fp8,
2697 amdgcn_smfmac_f32_16x16x128_fp8_bf8, amdgcn_smfmac_f32_16x16x128_fp8_fp8,
2698 amdgcn_smfmac_f32_32x32x64_bf8_bf8, amdgcn_smfmac_f32_32x32x64_bf8_fp8,
2699 amdgcn_smfmac_f32_32x32x64_fp8_bf8, amdgcn_smfmac_f32_32x32x64_fp8_fp8})
2700 .Any(Init: {.Predicate: {DivAnyTy},
2701 .OperandMapping: {{VgprOrAgprAnyTy},
2702 {IntrId, VgprAnyTy, VgprAnyTy, VgprOrAgprAnyTy, VgprAnyTy}}});
2703
2704 addRulesForIOpcs(OpcList: {amdgcn_mfma_scale_f32_32x32x64_f8f6f4,
2705 amdgcn_mfma_scale_f32_16x16x128_f8f6f4})
2706 .Any(Init: {.Predicate: {DivAnyTy},
2707 .OperandMapping: {{VgprOrAgprAnyTy},
2708 {IntrId, VgprAnyTy, VgprAnyTy, VgprOrAgprAnyTy, Imm, Imm, Imm,
2709 Vgpr32, Imm, Vgpr32}}});
2710
2711 // WMMA/SWMMAC intrinsics: all register operands map to VGPR.
2712 addRulesForIOpcs(
2713 OpcList: {// WMMA GFX11+
2714 amdgcn_wmma_f32_16x16x16_f16, amdgcn_wmma_f32_16x16x16_bf16,
2715 amdgcn_wmma_f16_16x16x16_f16, amdgcn_wmma_bf16_16x16x16_bf16,
2716 amdgcn_wmma_f16_16x16x16_f16_tied, amdgcn_wmma_bf16_16x16x16_bf16_tied,
2717 amdgcn_wmma_i32_16x16x16_iu8, amdgcn_wmma_i32_16x16x16_iu4,
2718 // WMMA GFX12
2719 amdgcn_wmma_f32_16x16x16_fp8_fp8, amdgcn_wmma_f32_16x16x16_fp8_bf8,
2720 amdgcn_wmma_f32_16x16x16_bf8_fp8, amdgcn_wmma_f32_16x16x16_bf8_bf8,
2721 amdgcn_wmma_i32_16x16x32_iu4,
2722 // WMMA GFX1250
2723 amdgcn_wmma_f32_16x16x4_f32, amdgcn_wmma_f32_16x16x32_bf16,
2724 amdgcn_wmma_f32_16x16x32_f16, amdgcn_wmma_f16_16x16x32_f16,
2725 amdgcn_wmma_bf16_16x16x32_bf16, amdgcn_wmma_bf16f32_16x16x32_bf16,
2726 amdgcn_wmma_f32_16x16x64_fp8_fp8, amdgcn_wmma_f32_16x16x64_fp8_bf8,
2727 amdgcn_wmma_f32_16x16x64_bf8_fp8, amdgcn_wmma_f32_16x16x64_bf8_bf8,
2728 amdgcn_wmma_f16_16x16x64_fp8_fp8, amdgcn_wmma_f16_16x16x64_fp8_bf8,
2729 amdgcn_wmma_f16_16x16x64_bf8_fp8, amdgcn_wmma_f16_16x16x64_bf8_bf8,
2730 amdgcn_wmma_f16_16x16x128_fp8_fp8, amdgcn_wmma_f16_16x16x128_fp8_bf8,
2731 amdgcn_wmma_f16_16x16x128_bf8_fp8, amdgcn_wmma_f16_16x16x128_bf8_bf8,
2732 amdgcn_wmma_f32_16x16x128_fp8_fp8, amdgcn_wmma_f32_16x16x128_fp8_bf8,
2733 amdgcn_wmma_f32_16x16x128_bf8_fp8, amdgcn_wmma_f32_16x16x128_bf8_bf8,
2734 amdgcn_wmma_i32_16x16x64_iu8, amdgcn_wmma_f32_16x16x128_f8f6f4,
2735 amdgcn_wmma_scale_f32_16x16x128_f8f6f4,
2736 amdgcn_wmma_scale16_f32_16x16x128_f8f6f4, amdgcn_wmma_f32_32x16x128_f4,
2737 amdgcn_wmma_scale_f32_32x16x128_f4, amdgcn_wmma_scale16_f32_32x16x128_f4,
2738 // WMMA GFX1251
2739 amdgcn_wmma_f64_16x16x4_f64,
2740 // SWMMAC GFX12
2741 amdgcn_swmmac_f32_16x16x32_f16, amdgcn_swmmac_f32_16x16x32_bf16,
2742 amdgcn_swmmac_f16_16x16x32_f16, amdgcn_swmmac_bf16_16x16x32_bf16,
2743 amdgcn_swmmac_i32_16x16x32_iu8, amdgcn_swmmac_i32_16x16x32_iu4,
2744 amdgcn_swmmac_i32_16x16x64_iu4, amdgcn_swmmac_f32_16x16x32_fp8_fp8,
2745 amdgcn_swmmac_f32_16x16x32_fp8_bf8, amdgcn_swmmac_f32_16x16x32_bf8_fp8,
2746 amdgcn_swmmac_f32_16x16x32_bf8_bf8,
2747 // SWMMAC GFX1250
2748 amdgcn_swmmac_f32_16x16x64_f16, amdgcn_swmmac_f32_16x16x64_bf16,
2749 amdgcn_swmmac_f16_16x16x64_f16, amdgcn_swmmac_bf16_16x16x64_bf16,
2750 amdgcn_swmmac_bf16f32_16x16x64_bf16, amdgcn_swmmac_f32_16x16x128_fp8_fp8,
2751 amdgcn_swmmac_f32_16x16x128_fp8_bf8, amdgcn_swmmac_f32_16x16x128_bf8_fp8,
2752 amdgcn_swmmac_f32_16x16x128_bf8_bf8, amdgcn_swmmac_f16_16x16x128_fp8_fp8,
2753 amdgcn_swmmac_f16_16x16x128_fp8_bf8, amdgcn_swmmac_f16_16x16x128_bf8_fp8,
2754 amdgcn_swmmac_f16_16x16x128_bf8_bf8, amdgcn_swmmac_i32_16x16x128_iu8})
2755 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {}, ApplyAllVgpr}});
2756
2757} // end initialize rules
2758