1//===-- AMDGPURegBankLegalizeRules.cpp ------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// Definitions of RegBankLegalize Rules for all opcodes.
10/// Implementation of container for all the Rules and search.
11/// Fast search for most common case when Rule.Predicate checks LLT and
12/// uniformity of register in operand 0.
13//
14//===----------------------------------------------------------------------===//
15
16#include "AMDGPURegBankLegalizeRules.h"
17#include "AMDGPUGlobalISelUtils.h"
18#include "AMDGPUInstrInfo.h"
19#include "GCNSubtarget.h"
20#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
21#include "llvm/CodeGen/MachineUniformityAnalysis.h"
22#include "llvm/IR/IntrinsicsAMDGPU.h"
23#include "llvm/Support/AMDGPUAddrSpace.h"
24
25#define DEBUG_TYPE "amdgpu-reg-bank-legalize"
26
27using namespace llvm;
28using namespace AMDGPU;
29
30bool AMDGPU::isAnyPtr(LLT Ty, unsigned Width) {
31 return Ty.isPointer() && Ty.getSizeInBits() == Width;
32}
33
34RegBankLLTMapping::RegBankLLTMapping(
35 std::initializer_list<RegBankLLTMappingApplyID> DstOpMappingList,
36 std::initializer_list<RegBankLLTMappingApplyID> SrcOpMappingList,
37 LoweringMethodID LoweringMethod)
38 : DstOpMapping(DstOpMappingList), SrcOpMapping(SrcOpMappingList),
39 LoweringMethod(LoweringMethod) {}
40
41PredicateMapping::PredicateMapping(
42 std::initializer_list<UniformityLLTOpPredicateID> OpList,
43 std::function<bool(const MachineInstr &)> TestFunc)
44 : OpUniformityAndTypes(OpList), TestFunc(TestFunc) {}
45
46bool matchUniformityAndLLT(Register Reg, UniformityLLTOpPredicateID UniID,
47 const MachineUniformityInfo &MUI,
48 const MachineRegisterInfo &MRI) {
49 switch (UniID) {
50 case S1:
51 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 1);
52 case S16:
53 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 16);
54 case S32:
55 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 32);
56 case S64:
57 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 64);
58 case S128:
59 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 128);
60 case P0:
61 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 0, SizeInBits: 64);
62 case P1:
63 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 1, SizeInBits: 64);
64 case P2:
65 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 2, SizeInBits: 32);
66 case P3:
67 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 3, SizeInBits: 32);
68 case P4:
69 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 4, SizeInBits: 64);
70 case P5:
71 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 5, SizeInBits: 32);
72 case P8:
73 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 8, SizeInBits: 128);
74 case Ptr32:
75 return isAnyPtr(Ty: MRI.getType(Reg), Width: 32);
76 case Ptr64:
77 return isAnyPtr(Ty: MRI.getType(Reg), Width: 64);
78 case Ptr128:
79 return isAnyPtr(Ty: MRI.getType(Reg), Width: 128);
80 case V2S16:
81 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16);
82 case V2S32:
83 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32);
84 case V3S32:
85 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 32);
86 case V4S32:
87 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32);
88 case B32:
89 return MRI.getType(Reg).getSizeInBits() == 32;
90 case B64:
91 return MRI.getType(Reg).getSizeInBits() == 64;
92 case B96:
93 return MRI.getType(Reg).getSizeInBits() == 96;
94 case B128:
95 return MRI.getType(Reg).getSizeInBits() == 128;
96 case B160:
97 return MRI.getType(Reg).getSizeInBits() == 160;
98 case B256:
99 return MRI.getType(Reg).getSizeInBits() == 256;
100 case B512:
101 return MRI.getType(Reg).getSizeInBits() == 512;
102 case DivAnyTy:
103 return MUI.isDivergentAtDef(V: Reg);
104 case UniS1:
105 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 1) && MUI.isUniformAtDef(V: Reg);
106 case UniS16:
107 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 16) && MUI.isUniformAtDef(V: Reg);
108 case UniS32:
109 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 32) && MUI.isUniformAtDef(V: Reg);
110 case UniS64:
111 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 64) && MUI.isUniformAtDef(V: Reg);
112 case UniS128:
113 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 128) && MUI.isUniformAtDef(V: Reg);
114 case UniBF16:
115 return MRI.getType(Reg).isBFloat16() && MUI.isUniformAtDef(V: Reg);
116 case UniP0:
117 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 0, SizeInBits: 64) && MUI.isUniformAtDef(V: Reg);
118 case UniP1:
119 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 1, SizeInBits: 64) && MUI.isUniformAtDef(V: Reg);
120 case UniP2:
121 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 2, SizeInBits: 32) && MUI.isUniformAtDef(V: Reg);
122 case UniP3:
123 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 3, SizeInBits: 32) && MUI.isUniformAtDef(V: Reg);
124 case UniP4:
125 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 4, SizeInBits: 64) && MUI.isUniformAtDef(V: Reg);
126 case UniP5:
127 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 5, SizeInBits: 32) && MUI.isUniformAtDef(V: Reg);
128 case UniP6:
129 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 6, SizeInBits: 32) && MUI.isUniformAtDef(V: Reg);
130 case UniP8:
131 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 8, SizeInBits: 128) && MUI.isUniformAtDef(V: Reg);
132 case UniPtr32:
133 return isAnyPtr(Ty: MRI.getType(Reg), Width: 32) && MUI.isUniformAtDef(V: Reg);
134 case UniPtr64:
135 return isAnyPtr(Ty: MRI.getType(Reg), Width: 64) && MUI.isUniformAtDef(V: Reg);
136 case UniPtr128:
137 return isAnyPtr(Ty: MRI.getType(Reg), Width: 128) && MUI.isUniformAtDef(V: Reg);
138 case UniV2S16:
139 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16) &&
140 MUI.isUniformAtDef(V: Reg);
141 case UniV2S32:
142 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32) &&
143 MUI.isUniformAtDef(V: Reg);
144 case UniV3S32:
145 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 32) &&
146 MUI.isUniformAtDef(V: Reg);
147 case UniV4S32:
148 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32) &&
149 MUI.isUniformAtDef(V: Reg);
150 case UniV6S32:
151 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 6, ScalarSizeInBits: 32) &&
152 MUI.isUniformAtDef(V: Reg);
153 case UniV8S16:
154 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16) &&
155 MUI.isUniformAtDef(V: Reg);
156 case UniV8S32:
157 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 32) &&
158 MUI.isUniformAtDef(V: Reg);
159 case UniV16S16:
160 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 16) &&
161 MUI.isUniformAtDef(V: Reg);
162 case UniV16S32:
163 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 32) &&
164 MUI.isUniformAtDef(V: Reg);
165 case UniV32S16:
166 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 32, ScalarSizeInBits: 16) &&
167 MUI.isUniformAtDef(V: Reg);
168 case UniV32S32:
169 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 32, ScalarSizeInBits: 32) &&
170 MUI.isUniformAtDef(V: Reg);
171 case UniV2S64:
172 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64) &&
173 MUI.isUniformAtDef(V: Reg);
174 case UniB32:
175 return MRI.getType(Reg).getSizeInBits() == 32 && MUI.isUniformAtDef(V: Reg);
176 case UniB64:
177 return MRI.getType(Reg).getSizeInBits() == 64 && MUI.isUniformAtDef(V: Reg);
178 case UniB96:
179 return MRI.getType(Reg).getSizeInBits() == 96 && MUI.isUniformAtDef(V: Reg);
180 case UniB128:
181 return MRI.getType(Reg).getSizeInBits() == 128 && MUI.isUniformAtDef(V: Reg);
182 case UniB160:
183 return MRI.getType(Reg).getSizeInBits() == 160 && MUI.isUniformAtDef(V: Reg);
184 case UniB256:
185 return MRI.getType(Reg).getSizeInBits() == 256 && MUI.isUniformAtDef(V: Reg);
186 case UniB512:
187 return MRI.getType(Reg).getSizeInBits() == 512 && MUI.isUniformAtDef(V: Reg);
188 case UniBRC: {
189 if (MUI.isDivergentAtDef(V: Reg))
190 return false;
191 // Check if there is SGPR register class of same size as the LLT.
192 const SIRegisterInfo *TRI =
193 static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
194 // There is no 16 bit SGPR register class. Extra size check is required
195 // since getSGPRClassForBitWidth returns SReg_32RegClass for Size 16.
196 unsigned LLTSize = MRI.getType(Reg).getSizeInBits();
197 return LLTSize >= 32 && TRI->getSGPRClassForBitWidth(BitWidth: LLTSize);
198 }
199 case DivS1:
200 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 1) && MUI.isDivergentAtDef(V: Reg);
201 case DivS16:
202 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 16) && MUI.isDivergentAtDef(V: Reg);
203 case DivS32:
204 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 32) && MUI.isDivergentAtDef(V: Reg);
205 case DivS64:
206 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 64) && MUI.isDivergentAtDef(V: Reg);
207 case DivS128:
208 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 128) && MUI.isDivergentAtDef(V: Reg);
209 case DivP0:
210 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 0, SizeInBits: 64) && MUI.isDivergentAtDef(V: Reg);
211 case DivP1:
212 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 1, SizeInBits: 64) && MUI.isDivergentAtDef(V: Reg);
213 case DivP2:
214 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 2, SizeInBits: 32) && MUI.isDivergentAtDef(V: Reg);
215 case DivP3:
216 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 3, SizeInBits: 32) && MUI.isDivergentAtDef(V: Reg);
217 case DivP4:
218 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 4, SizeInBits: 64) && MUI.isDivergentAtDef(V: Reg);
219 case DivP5:
220 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 5, SizeInBits: 32) && MUI.isDivergentAtDef(V: Reg);
221 case DivPtr32:
222 return isAnyPtr(Ty: MRI.getType(Reg), Width: 32) && MUI.isDivergentAtDef(V: Reg);
223 case DivPtr64:
224 return isAnyPtr(Ty: MRI.getType(Reg), Width: 64) && MUI.isDivergentAtDef(V: Reg);
225 case DivPtr128:
226 return isAnyPtr(Ty: MRI.getType(Reg), Width: 128) && MUI.isDivergentAtDef(V: Reg);
227 case DivV2S16:
228 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16) &&
229 MUI.isDivergentAtDef(V: Reg);
230 case DivV2S32:
231 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32) &&
232 MUI.isDivergentAtDef(V: Reg);
233 case DivV4S32:
234 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32) &&
235 MUI.isDivergentAtDef(V: Reg);
236 case DivV2S64:
237 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64) &&
238 MUI.isDivergentAtDef(V: Reg);
239 case DivV3S32:
240 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 32) &&
241 MUI.isDivergentAtDef(V: Reg);
242 case DivV4S16:
243 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16) &&
244 MUI.isDivergentAtDef(V: Reg);
245 case DivV8S16:
246 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16) &&
247 MUI.isDivergentAtDef(V: Reg);
248 case DivV8S32:
249 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 32) &&
250 MUI.isDivergentAtDef(V: Reg);
251 case DivV16S16:
252 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 16) &&
253 MUI.isDivergentAtDef(V: Reg);
254 case DivV16S32:
255 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 32) &&
256 MUI.isDivergentAtDef(V: Reg);
257 case DivV6S32:
258 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 6, ScalarSizeInBits: 32) &&
259 MUI.isDivergentAtDef(V: Reg);
260 case DivV32S16:
261 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 32, ScalarSizeInBits: 16) &&
262 MUI.isDivergentAtDef(V: Reg);
263 case DivV32S32:
264 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 32, ScalarSizeInBits: 32) &&
265 MUI.isDivergentAtDef(V: Reg);
266 case DivB32:
267 return MRI.getType(Reg).getSizeInBits() == 32 && MUI.isDivergentAtDef(V: Reg);
268 case DivB64:
269 return MRI.getType(Reg).getSizeInBits() == 64 && MUI.isDivergentAtDef(V: Reg);
270 case DivB96:
271 return MRI.getType(Reg).getSizeInBits() == 96 && MUI.isDivergentAtDef(V: Reg);
272 case DivB128:
273 return MRI.getType(Reg).getSizeInBits() == 128 && MUI.isDivergentAtDef(V: Reg);
274 case DivB160:
275 return MRI.getType(Reg).getSizeInBits() == 160 && MUI.isDivergentAtDef(V: Reg);
276 case DivB256:
277 return MRI.getType(Reg).getSizeInBits() == 256 && MUI.isDivergentAtDef(V: Reg);
278 case DivB512:
279 return MRI.getType(Reg).getSizeInBits() == 512 && MUI.isDivergentAtDef(V: Reg);
280 case DivBRC: {
281 if (MUI.isUniformAtDef(V: Reg))
282 return false;
283 // Check if there is VGPR register class of same size as the LLT.
284 const SIRegisterInfo *TRI =
285 static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
286 return TRI->getSGPRClassForBitWidth(BitWidth: MRI.getType(Reg).getSizeInBits());
287 }
288 case BRC: {
289 // Check if there is SGPR and VGPR register class of same size as the LLT.
290 const SIRegisterInfo *TRI =
291 static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
292 unsigned LLTSize = MRI.getType(Reg).getSizeInBits();
293 return LLTSize >= 32 && TRI->getSGPRClassForBitWidth(BitWidth: LLTSize) &&
294 TRI->getVGPRClassForBitWidth(BitWidth: LLTSize);
295 }
296 case _:
297 return true;
298 default:
299 llvm_unreachable("missing matchUniformityAndLLT");
300 }
301}
302
303bool PredicateMapping::match(const MachineInstr &MI,
304 const MachineUniformityInfo &MUI,
305 const MachineRegisterInfo &MRI) const {
306 // Check LLT signature.
307 for (unsigned i = 0; i < OpUniformityAndTypes.size(); ++i) {
308 const MachineOperand &MO = MI.getOperand(i);
309 if (OpUniformityAndTypes[i] == _) {
310 assert((!MI.getOperand(i).isReg() ||
311 !MI.getOperand(i).getReg().isVirtual()) &&
312 "_ is for non-register and physical register operands only");
313 continue;
314 }
315
316 // Remaining IDs check registers.
317 if (!MO.isReg())
318 return false;
319
320 if (!matchUniformityAndLLT(Reg: MO.getReg(), UniID: OpUniformityAndTypes[i], MUI, MRI))
321 return false;
322 }
323
324 // More complex check.
325 if (TestFunc)
326 return TestFunc(MI);
327
328 return true;
329}
330
331SetOfRulesForOpcode::SetOfRulesForOpcode() = default;
332
333SetOfRulesForOpcode::SetOfRulesForOpcode(FastRulesTypes FastTypes)
334 : FastTypes(FastTypes) {}
335
336UniformityLLTOpPredicateID LLTToId(LLT Ty) {
337 if (Ty == LLT::scalar(SizeInBits: 16))
338 return S16;
339 if (Ty == LLT::scalar(SizeInBits: 32))
340 return S32;
341 if (Ty == LLT::scalar(SizeInBits: 64))
342 return S64;
343 if (Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16))
344 return V2S16;
345 if (Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32))
346 return V2S32;
347 if (Ty == LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 32))
348 return V3S32;
349 if (Ty == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32))
350 return V4S32;
351 return _;
352}
353
354UniformityLLTOpPredicateID LLTToBId(LLT Ty) {
355 if (Ty == LLT::scalar(SizeInBits: 32) || Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16) ||
356 isAnyPtr(Ty, Width: 32))
357 return B32;
358 if (Ty == LLT::scalar(SizeInBits: 64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32) ||
359 Ty == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16) || isAnyPtr(Ty, Width: 64))
360 return B64;
361 if (Ty == LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 32))
362 return B96;
363 if (Ty == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32) || Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64) ||
364 Ty == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16) || isAnyPtr(Ty, Width: 128))
365 return B128;
366 return _;
367}
368
369const RegBankLLTMapping *
370SetOfRulesForOpcode::findMappingForMI(const MachineInstr &MI,
371 const MachineRegisterInfo &MRI,
372 const MachineUniformityInfo &MUI) const {
373 // Search in "Fast Rules".
374 // Note: if fast rules are enabled, RegBankLLTMapping must be added in each
375 // slot that could "match fast Predicate". If not, InvalidMapping is
376 // returned which results in failure, does not search "Slow Rules".
377 if (FastTypes != NoFastRules) {
378 Register Reg = MI.getOperand(i: 0).getReg();
379 int Slot;
380 if (FastTypes == StandardB)
381 Slot = getFastPredicateSlot(Ty: LLTToBId(Ty: MRI.getType(Reg)));
382 else
383 Slot = getFastPredicateSlot(Ty: LLTToId(Ty: MRI.getType(Reg)));
384
385 if (Slot != -1)
386 return MUI.isUniformAtDef(V: Reg) ? &Uni[Slot] : &Div[Slot];
387 }
388
389 // Slow search for more complex rules.
390 for (const RegBankLegalizeRule &Rule : Rules) {
391 if (Rule.Predicate.match(MI, MUI, MRI))
392 return &Rule.OperandMapping;
393 }
394
395 return nullptr;
396}
397
398void SetOfRulesForOpcode::addRule(RegBankLegalizeRule Rule) {
399 Rules.push_back(Elt: Rule);
400}
401
402void SetOfRulesForOpcode::addFastRuleDivergent(UniformityLLTOpPredicateID Ty,
403 RegBankLLTMapping RuleApplyIDs) {
404 int Slot = getFastPredicateSlot(Ty);
405 assert(Slot != -1 && "Ty unsupported in this FastRulesTypes");
406 Div[Slot] = std::move(RuleApplyIDs);
407}
408
409void SetOfRulesForOpcode::addFastRuleUniform(UniformityLLTOpPredicateID Ty,
410 RegBankLLTMapping RuleApplyIDs) {
411 int Slot = getFastPredicateSlot(Ty);
412 assert(Slot != -1 && "Ty unsupported in this FastRulesTypes");
413 Uni[Slot] = std::move(RuleApplyIDs);
414}
415
416int SetOfRulesForOpcode::getFastPredicateSlot(
417 UniformityLLTOpPredicateID Ty) const {
418 switch (FastTypes) {
419 case Standard: {
420 switch (Ty) {
421 case S32:
422 return 0;
423 case S16:
424 return 1;
425 case S64:
426 return 2;
427 case V2S16:
428 return 3;
429 default:
430 return -1;
431 }
432 }
433 case StandardB: {
434 switch (Ty) {
435 case B32:
436 return 0;
437 case B64:
438 return 1;
439 case B96:
440 return 2;
441 case B128:
442 return 3;
443 default:
444 return -1;
445 }
446 }
447 case Vector: {
448 switch (Ty) {
449 case S32:
450 return 0;
451 case V2S32:
452 return 1;
453 case V3S32:
454 return 2;
455 case V4S32:
456 return 3;
457 default:
458 return -1;
459 }
460 }
461 default:
462 return -1;
463 }
464}
465
466RegBankLegalizeRules::RuleSetInitializer
467RegBankLegalizeRules::addRulesForGOpcs(std::initializer_list<unsigned> OpcList,
468 FastRulesTypes FastTypes) {
469 return RuleSetInitializer(OpcList, GRulesAlias, GRules, FastTypes);
470}
471
472RegBankLegalizeRules::RuleSetInitializer
473RegBankLegalizeRules::addRulesForIOpcs(std::initializer_list<unsigned> OpcList,
474 FastRulesTypes FastTypes) {
475 return RuleSetInitializer(OpcList, IRulesAlias, IRules, FastTypes);
476}
477
478const SetOfRulesForOpcode *
479RegBankLegalizeRules::getRulesForOpc(MachineInstr &MI) const {
480 unsigned Opc = MI.getOpcode();
481 if (Opc == AMDGPU::G_INTRINSIC || Opc == AMDGPU::G_INTRINSIC_CONVERGENT ||
482 Opc == AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS ||
483 Opc == AMDGPU::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS) {
484 unsigned IntrID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
485 auto IRAIt = IRulesAlias.find(Val: IntrID);
486 if (IRAIt == IRulesAlias.end())
487 return nullptr;
488 return &IRules.at(Val: IRAIt->second);
489 }
490
491 auto GRAIt = GRulesAlias.find(Val: Opc);
492 if (GRAIt == GRulesAlias.end())
493 return nullptr;
494 return &GRules.at(Val: GRAIt->second);
495}
496
497// Syntactic sugar wrapper for predicate lambda that enables '&&', '||' and '!'.
498class Predicate {
499private:
500 struct Elt {
501 // Save formula composed of Pred, '&&', '||' and '!' as a jump table.
502 // Sink ! to Pred. For example !((A && !B) || C) -> (!A || B) && !C
503 // Sequences of && and || will be represented by jumps, for example:
504 // (A && B && ... X) or (A && B && ... X) || Y
505 // A == true jump to B
506 // A == false jump to end or Y, result is A(false) or Y
507 // (A || B || ... X) or (A || B || ... X) && Y
508 // A == true jump to end or Y, result is A(true) or Y
509 // A == false jump to B
510 // Notice that when negating expression, we simply flip Neg on each Pred
511 // and swap TJumpOffset and FJumpOffset (&& becomes ||, || becomes &&).
512 std::function<bool(const MachineInstr &)> Pred;
513 bool Neg; // Neg of Pred is calculated before jump
514 unsigned TJumpOffset;
515 unsigned FJumpOffset;
516 };
517
518 SmallVector<Elt, 8> Expression;
519
520 Predicate(SmallVectorImpl<Elt> &&Expr) { Expression.swap(RHS&: Expr); };
521
522public:
523 Predicate(std::function<bool(const MachineInstr &)> Pred) {
524 Expression.push_back(Elt: {.Pred: Pred, .Neg: false, .TJumpOffset: 1, .FJumpOffset: 1});
525 };
526
527 bool operator()(const MachineInstr &MI) const {
528 unsigned Idx = 0;
529 unsigned ResultIdx = Expression.size();
530 bool Result;
531 do {
532 Result = Expression[Idx].Pred(MI);
533 Result = Expression[Idx].Neg ? !Result : Result;
534 if (Result) {
535 Idx += Expression[Idx].TJumpOffset;
536 } else {
537 Idx += Expression[Idx].FJumpOffset;
538 }
539 } while ((Idx != ResultIdx));
540
541 return Result;
542 };
543
544 Predicate operator!() const {
545 SmallVector<Elt, 8> NegExpression;
546 for (const Elt &ExprElt : Expression) {
547 NegExpression.push_back(Elt: {.Pred: ExprElt.Pred, .Neg: !ExprElt.Neg, .TJumpOffset: ExprElt.FJumpOffset,
548 .FJumpOffset: ExprElt.TJumpOffset});
549 }
550 return Predicate(std::move(NegExpression));
551 };
552
553 Predicate operator&&(const Predicate &RHS) const {
554 SmallVector<Elt, 8> AndExpression = Expression;
555
556 unsigned RHSSize = RHS.Expression.size();
557 unsigned ResultIdx = Expression.size();
558 for (unsigned i = 0; i < ResultIdx; ++i) {
559 // LHS results in false, whole expression results in false.
560 if (i + AndExpression[i].FJumpOffset == ResultIdx)
561 AndExpression[i].FJumpOffset += RHSSize;
562 }
563
564 AndExpression.append(RHS: RHS.Expression);
565
566 return Predicate(std::move(AndExpression));
567 }
568
569 Predicate operator||(const Predicate &RHS) const {
570 SmallVector<Elt, 8> OrExpression = Expression;
571
572 unsigned RHSSize = RHS.Expression.size();
573 unsigned ResultIdx = Expression.size();
574 for (unsigned i = 0; i < ResultIdx; ++i) {
575 // LHS results in true, whole expression results in true.
576 if (i + OrExpression[i].TJumpOffset == ResultIdx)
577 OrExpression[i].TJumpOffset += RHSSize;
578 }
579
580 OrExpression.append(RHS: RHS.Expression);
581
582 return Predicate(std::move(OrExpression));
583 }
584};
585
586// Initialize rules
587RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
588 MachineRegisterInfo &_MRI)
589 : ST(&_ST), MRI(&_MRI) {
590
591 addRulesForGOpcs(OpcList: {G_ADD, G_SUB}, FastTypes: Standard)
592 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr32Trunc}, {Sgpr32AExt, Sgpr32AExt}})
593 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
594 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}})
595 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
596 .Uni(Ty: V2S16, RuleApplyIDs: {{SgprV2S16}, {SgprV2S16, SgprV2S16}, UnpackAExt})
597 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
598 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {Sgpr64, Sgpr64}})
599 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr64}})
600 .Any(Init: {.Predicate: {UniV2S64}, .OperandMapping: {{UniInVgprV2S64}, {VgprV2S64, VgprV2S64}}})
601 .Any(Init: {.Predicate: {DivV2S64}, .OperandMapping: {{VgprV2S64}, {VgprV2S64, VgprV2S64}}});
602
603 addRulesForGOpcs(OpcList: {G_UADDO, G_USUBO}, FastTypes: Standard)
604 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32, Sgpr32Trunc}, {Sgpr32, Sgpr32}})
605 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32, Vcc}, {Vgpr32, Vgpr32}});
606
607 addRulesForGOpcs(OpcList: {G_UADDE, G_USUBE, G_SADDE, G_SSUBE}, FastTypes: Standard)
608 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32, Sgpr32Trunc}, {Sgpr32, Sgpr32, Sgpr32AExtBoolInReg}})
609 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32, Vcc}, {Vgpr32, Vgpr32, Vcc}});
610
611 addRulesForGOpcs(OpcList: {G_UADDSAT, G_SADDSAT, G_USUBSAT, G_SSUBSAT}, FastTypes: Standard)
612 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16, Vgpr16}})
613 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
614 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32}})
615 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
616 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16}})
617 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}});
618
619 bool UseVecMulU64 = ST->useVMulU64Inst();
620 addRulesForGOpcs(OpcList: {G_MUL}, FastTypes: Standard)
621 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
622 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}})
623 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
624 .Uni(Ty: S64, RuleApplyIDs: {{SgprB64}, {SgprB64, SgprB64}})
625 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16}})
626 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
627 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr32Trunc}, {Sgpr32AExt, Sgpr32AExt}})
628 .Div(Ty: S64, RuleApplyIDs: {{VgprB64}, {VgprB64, VgprB64}}, STPred: UseVecMulU64)
629 .Div(Ty: S64, RuleApplyIDs: {{VgprB64}, {VgprB64, VgprB64}, SplitTo32Mul}, STPred: !UseVecMulU64);
630
631 bool hasMulHi = ST->hasScalarMulHiInsts();
632 addRulesForGOpcs(OpcList: {G_UMULH, G_SMULH}, FastTypes: Standard)
633 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
634 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}}, STPred: hasMulHi)
635 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32}}, STPred: !hasMulHi);
636
637 addRulesForGOpcs(OpcList: {G_AMDGPU_MAD_U64_U32}, FastTypes: Standard)
638 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64, Vcc}, {Vgpr32, Vgpr32, Vgpr64}})
639 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64, Sgpr32Trunc}, {Sgpr32, Sgpr32, Sgpr64}, UniMAD64});
640
641 bool HasScalarSMulU64 = ST->hasScalarSMulU64();
642 addRulesForGOpcs(OpcList: {G_AMDGPU_S_MUL_U64_U32, G_AMDGPU_S_MUL_I64_I32}, FastTypes: Standard)
643 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {Sgpr64, Sgpr64}, UniMul64}, STPred: HasScalarSMulU64)
644 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr64}, DivSMulToMAD});
645
646 addRulesForGOpcs(OpcList: {G_XOR, G_OR, G_AND}, FastTypes: StandardB)
647 .Any(Init: {.Predicate: {UniS1}, .OperandMapping: {{Sgpr32Trunc}, {Sgpr32AExt, Sgpr32AExt}}})
648 .Any(Init: {.Predicate: {DivS1}, .OperandMapping: {{Vcc}, {Vcc, Vcc}}})
649 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{Sgpr16}, {Sgpr16, Sgpr16}}})
650 .Any(Init: {.Predicate: {DivS16}, .OperandMapping: {{Vgpr16}, {Vgpr16, Vgpr16}}})
651 .Uni(Ty: B32, RuleApplyIDs: {{SgprB32}, {SgprB32, SgprB32}})
652 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {VgprB32, VgprB32}})
653 .Uni(Ty: B64, RuleApplyIDs: {{SgprB64}, {SgprB64, SgprB64}})
654 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {VgprB64, VgprB64}, SplitTo32});
655
656 addRulesForGOpcs(OpcList: {G_SHL}, FastTypes: Standard)
657 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr32Trunc}, {Sgpr32AExt, Sgpr32ZExt}})
658 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
659 .Uni(Ty: V2S16, RuleApplyIDs: {{SgprV2S16}, {SgprV2S16, SgprV2S16}, UnpackBitShift})
660 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
661 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}})
662 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {Sgpr64, Sgpr32}})
663 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
664 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr32}});
665
666 addRulesForGOpcs(OpcList: {G_LSHR}, FastTypes: Standard)
667 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr32Trunc}, {Sgpr32ZExt, Sgpr32ZExt}})
668 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
669 .Uni(Ty: V2S16, RuleApplyIDs: {{SgprV2S16}, {SgprV2S16, SgprV2S16}, UnpackBitShift})
670 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
671 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}})
672 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {Sgpr64, Sgpr32}})
673 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
674 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr32}});
675
676 addRulesForGOpcs(OpcList: {G_ASHR}, FastTypes: Standard)
677 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr32Trunc}, {Sgpr32SExt, Sgpr32ZExt}})
678 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
679 .Uni(Ty: V2S16, RuleApplyIDs: {{SgprV2S16}, {SgprV2S16, SgprV2S16}, UnpackBitShift})
680 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
681 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}})
682 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {Sgpr64, Sgpr32}})
683 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
684 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr32}});
685
686 addRulesForGOpcs(OpcList: {G_FSHR}, FastTypes: Standard)
687 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32, Sgpr32}})
688 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32, Vgpr32}});
689
690 addRulesForGOpcs(OpcList: {G_BSWAP}, FastTypes: Standard)
691 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16}})
692 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16}})
693 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32}})
694 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32}})
695 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {VgprV2S16}})
696 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16}});
697
698 addRulesForGOpcs(OpcList: {G_AMDGPU_CVT_F32_UBYTE0, G_AMDGPU_CVT_F32_UBYTE1,
699 G_AMDGPU_CVT_F32_UBYTE2, G_AMDGPU_CVT_F32_UBYTE3,
700 G_AMDGPU_RCP_IFLAG},
701 FastTypes: Standard)
702 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32}})
703 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32}});
704
705 addRulesForGOpcs(OpcList: {G_FRAME_INDEX}).Any(Init: {.Predicate: {UniP5, _}, .OperandMapping: {{SgprP5}, {None}}});
706
707 addRulesForGOpcs(OpcList: {G_UBFX, G_SBFX}, FastTypes: Standard)
708 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32, Sgpr32}, S_BFE})
709 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32, Vgpr32}})
710 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {Sgpr64, Sgpr32, Sgpr32}, S_BFE})
711 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr32, Vgpr32}, V_BFE});
712
713 addRulesForGOpcs(OpcList: {G_SMIN, G_SMAX}, FastTypes: Standard)
714 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr32Trunc}, {Sgpr32SExt, Sgpr32SExt}})
715 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
716 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}})
717 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
718 .Uni(Ty: V2S16, RuleApplyIDs: {{SgprV2S16}, {SgprV2S16, SgprV2S16}, UnpackMinMax})
719 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
720 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64, Vgpr64}})
721 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr64}});
722
723 addRulesForGOpcs(OpcList: {G_UMIN, G_UMAX}, FastTypes: Standard)
724 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr32Trunc}, {Sgpr32ZExt, Sgpr32ZExt}})
725 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
726 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}})
727 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
728 .Uni(Ty: V2S16, RuleApplyIDs: {{SgprV2S16}, {SgprV2S16, SgprV2S16}, UnpackMinMax})
729 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
730 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64, Vgpr64}})
731 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr64}});
732
733 addRulesForGOpcs(OpcList: {G_IMPLICIT_DEF})
734 .Any(Init: {.Predicate: {UniS1}, .OperandMapping: {{Sgpr32Trunc}, {}}})
735 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{Sgpr16}, {}}})
736 .Any(Init: {.Predicate: {UniBRC}, .OperandMapping: {{SgprBRC}, {}}});
737
738 addRulesForGOpcs(OpcList: {G_CONSTANT}, FastTypes: Standard)
739 .Any(Init: {.Predicate: {UniS1, _}, .OperandMapping: {{Sgpr32Trunc}, {}, UniCstExt}})
740 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {}})
741 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {}})
742 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {}})
743 .Any(Init: {.Predicate: {UniPtr32, _}, .OperandMapping: {{SgprPtr32}, {}}})
744 .Any(Init: {.Predicate: {UniPtr64, _}, .OperandMapping: {{SgprPtr64}, {}}});
745
746 addRulesForGOpcs(OpcList: {G_FCONSTANT}, FastTypes: Standard)
747 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {}})
748 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {}})
749 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {}});
750
751 addRulesForGOpcs(OpcList: {G_FREEZE})
752 .Any(Init: {.Predicate: {UniS1}, .OperandMapping: {{Sgpr32Trunc}, {Sgpr32AExt}}})
753 .Any(Init: {.Predicate: {DivS1}, .OperandMapping: {{Vcc}, {Vcc}}})
754 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{Sgpr16}, {Sgpr16}}})
755 .Any(Init: {.Predicate: {UniBRC}, .OperandMapping: {{SgprBRC}, {SgprBRC}}})
756 .Any(Init: {.Predicate: {DivBRC}, .OperandMapping: {{VgprBRC}, {VgprBRC}}});
757
758 addRulesForGOpcs(OpcList: {G_BITCAST})
759 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{Sgpr16}, {Sgpr16}}})
760 .Any(Init: {.Predicate: {DivS16}, .OperandMapping: {{Vgpr16}, {Vgpr16}}})
761 .Any(Init: {.Predicate: {UniBRC}, .OperandMapping: {{SgprBRC}, {SgprBRC}}})
762 .Any(Init: {.Predicate: {DivBRC}, .OperandMapping: {{VgprBRC}, {VgprBRC}}});
763
764 addRulesForGOpcs(OpcList: {G_UNMERGE_VALUES})
765 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{}, {}, UnmergeToShiftTrunc}})
766 .Any(Init: {.Predicate: {UniBRC}, .OperandMapping: {{}, {}, VerifyAllSgpr}})
767 .Any(Init: {.Predicate: {DivBRC}, .OperandMapping: {{}, {}, ApplyAllVgpr}});
768
769 addRulesForGOpcs(OpcList: {G_BUILD_VECTOR, G_MERGE_VALUES})
770 .Any(Init: {.Predicate: {UniBRC, S16}, .OperandMapping: {{}, {}, VerifyAllSgpr}})
771 .Any(Init: {.Predicate: {UniBRC, BRC}, .OperandMapping: {{}, {}, VerifyAllSgpr}})
772 .Any(Init: {.Predicate: {DivBRC, S16}, .OperandMapping: {{}, {}, ApplyAllVgpr}})
773 .Any(Init: {.Predicate: {DivBRC, BRC}, .OperandMapping: {{}, {}, ApplyAllVgpr}});
774
775 addRulesForGOpcs(OpcList: {G_CONCAT_VECTORS})
776 .Any(Init: {.Predicate: {UniBRC, BRC}, .OperandMapping: {{}, {}, VerifyAllSgpr}})
777 .Any(Init: {.Predicate: {DivBRC, BRC}, .OperandMapping: {{}, {}, ApplyAllVgpr}});
778
779 addRulesForGOpcs(OpcList: {G_PHI})
780 .Any(Init: {.Predicate: {UniS1}, .OperandMapping: {{}, {}, AextToS32InIncomingBlockGPHI}})
781 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{}, {}, VerifyAllSgprGPHI}})
782 .Any(Init: {.Predicate: {UniBRC}, .OperandMapping: {{}, {}, VerifyAllSgprGPHI}})
783 .Any(Init: {.Predicate: {DivBRC}, .OperandMapping: {{}, {}, VerifyAllSgprOrVgprGPHI}});
784
785 addRulesForGOpcs(OpcList: {G_EXTRACT_VECTOR_ELT})
786 .Any(Init: {.Predicate: {UniB32, UniBRC, UniS32}, .OperandMapping: {{SgprB32}, {SgprBRC, Sgpr32}}})
787 .Any(Init: {.Predicate: {DivB32, DivBRC, UniS32}, .OperandMapping: {{VgprB32}, {VgprBRC, Sgpr32}}})
788 .Any(Init: {.Predicate: {DivB32, BRC, DivS32},
789 .OperandMapping: {{VgprB32}, {VgprBRC, Vgpr32}, ExtrVecEltToSel}})
790 .Any(Init: {.Predicate: {UniB64, UniBRC, UniS32}, .OperandMapping: {{SgprB64}, {SgprBRC, Sgpr32}}})
791 .Any(Init: {.Predicate: {DivB64, DivBRC, UniS32},
792 .OperandMapping: {{VgprB64}, {VgprBRC, Sgpr32}, ExtrVecEltTo32}})
793 .Any(Init: {.Predicate: {DivB64, BRC, DivS32},
794 .OperandMapping: {{VgprB64}, {VgprBRC, Vgpr32}, ExtrVecEltToSel}});
795
796 addRulesForGOpcs(OpcList: {G_INSERT_VECTOR_ELT})
797 .Any(Init: {.Predicate: {UniBRC, UniBRC, UniB32, UniS32},
798 .OperandMapping: {{SgprBRC}, {SgprBRC, SgprB32, Sgpr32}}})
799 .Any(
800 Init: {.Predicate: {DivBRC, BRC, B32, UniS32}, .OperandMapping: {{VgprBRC}, {VgprBRC, VgprB32, Sgpr32}}})
801 .Any(Init: {.Predicate: {DivBRC, BRC, B32, DivS32},
802 .OperandMapping: {{VgprBRC}, {VgprBRC, VgprB32, Vgpr32}, InsVecEltToSel}})
803 .Any(Init: {.Predicate: {UniBRC, UniBRC, UniB64, UniS32},
804 .OperandMapping: {{SgprBRC}, {SgprBRC, SgprB64, Sgpr32}, InsVecEltToSel}})
805 .Any(Init: {.Predicate: {DivBRC, BRC, B64, UniS32},
806 .OperandMapping: {{VgprBRC}, {VgprBRC, VgprB64, Sgpr32}, InsVecEltTo32}})
807 .Any(Init: {.Predicate: {DivBRC, BRC, B64, DivS32},
808 .OperandMapping: {{VgprBRC}, {VgprBRC, VgprB64, Vgpr32}, InsVecEltToSel}});
809
810 // INTERSECT_RAY {Div}, {{VgprDst...}, {VgprSrc, ..., Sgpr_WF_RsrcIdx}}
811 // INTERSECT_RAY {Uni}, {{UniInVgprDst...}, {VgprSrc, ..., Sgpr_WF_RsrcIdx}}
812 addRulesForGOpcs(OpcList: {G_AMDGPU_BVH_INTERSECT_RAY, G_AMDGPU_BVH_DUAL_INTERSECT_RAY,
813 G_AMDGPU_BVH8_INTERSECT_RAY})
814 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {}, ApplyBVH_INTERSECT_RAY}});
815
816 // LOAD {Div}, {{VgprDst...}, {VgprSrc, ..., Sgpr_WF_RsrcIdx}}
817 // LOAD {Uni}, {{UniInVgprDst...}, {VgprSrc, ..., Sgpr_WF_RsrcIdx}}
818 // LOAD_NORET {}, {{}, {Imm, VgprSrc, ..., Sgpr_WF_RsrcIdx}}
819 // STORE {}, {{}, {VgprSrc, ..., Sgpr_WF_RsrcIdx}}
820 addRulesForGOpcs(OpcList: {G_AMDGPU_INTRIN_IMAGE_LOAD, G_AMDGPU_INTRIN_IMAGE_LOAD_D16,
821 G_AMDGPU_INTRIN_IMAGE_LOAD_NORET,
822 G_AMDGPU_INTRIN_IMAGE_STORE,
823 G_AMDGPU_INTRIN_IMAGE_STORE_D16})
824 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {}, ApplyINTRIN_IMAGE}});
825
826 Predicate isSignedICmp([](const MachineInstr &MI) -> bool {
827 auto Pred =
828 static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
829 return CmpInst::isSigned(Pred);
830 });
831
832 Predicate isEqualityICmp([](const MachineInstr &MI) -> bool {
833 auto Pred =
834 static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
835 return ICmpInst::isEquality(P: Pred);
836 });
837
838 bool HasScalarCompareEq64 = ST->hasScalarCompareEq64();
839 // clang-format off
840 addRulesForGOpcs(OpcList: {G_ICMP})
841 .Any(Init: {.Predicate: {{UniS1, _, S16}, isEqualityICmp}, .OperandMapping: {{Sgpr32Trunc}, {None, Sgpr32ZExt, Sgpr32ZExt}}})
842 .Any(Init: {.Predicate: {{UniS1, _, S16}, !isEqualityICmp && isSignedICmp}, .OperandMapping: {{Sgpr32Trunc}, {None, Sgpr32SExt, Sgpr32SExt}}})
843 .Any(Init: {.Predicate: {{UniS1, _, S16}, !isEqualityICmp && !isSignedICmp}, .OperandMapping: {{Sgpr32Trunc}, {None, Sgpr32ZExt, Sgpr32ZExt}}})
844 .Any(Init: {.Predicate: {{DivS1, _, S16}}, .OperandMapping: {{Vcc}, {None, Vgpr16, Vgpr16}}})
845 .Any(Init: {.Predicate: {{UniS1, _, S32}}, .OperandMapping: {{Sgpr32Trunc}, {None, Sgpr32, Sgpr32}}})
846 .Any(Init: {.Predicate: {{DivS1, _, S32}}, .OperandMapping: {{Vcc}, {None, Vgpr32, Vgpr32}}})
847 .Any(Init: {.Predicate: {{UniS1, _, S64}, isEqualityICmp}, .OperandMapping: {{Sgpr32Trunc}, {None, Sgpr64, Sgpr64}}}, STPred: HasScalarCompareEq64)
848 .Any(Init: {.Predicate: {{UniS1, _, S64}, isEqualityICmp}, .OperandMapping: {{UniInVcc}, {None, Vgpr64, Vgpr64}}}, STPred: !HasScalarCompareEq64)
849 .Any(Init: {.Predicate: {{UniS1, _, S64}, !isEqualityICmp}, .OperandMapping: {{UniInVcc}, {None, Vgpr64, Vgpr64}}})
850 .Any(Init: {.Predicate: {{DivS1, _, S64}}, .OperandMapping: {{Vcc}, {None, Vgpr64, Vgpr64}}})
851 .Any(Init: {.Predicate: {{UniS1, _, Ptr32}}, .OperandMapping: {{Sgpr32Trunc}, {None, SgprPtr32, SgprPtr32}}})
852 .Any(Init: {.Predicate: {{DivS1, _, Ptr32}}, .OperandMapping: {{Vcc}, {None, VgprPtr32, VgprPtr32}}})
853 .Any(Init: {.Predicate: {{UniS1, _, Ptr64}, isEqualityICmp}, .OperandMapping: {{Sgpr32Trunc}, {None, SgprPtr64, SgprPtr64}}}, STPred: HasScalarCompareEq64)
854 .Any(Init: {.Predicate: {{UniS1, _, Ptr64}, isEqualityICmp}, .OperandMapping: {{UniInVcc}, {None, VgprPtr64, VgprPtr64}}}, STPred: !HasScalarCompareEq64)
855 .Any(Init: {.Predicate: {{UniS1, _, Ptr64}, !isEqualityICmp}, .OperandMapping: {{UniInVcc}, {None, VgprPtr64, VgprPtr64}}})
856 .Any(Init: {.Predicate: {{DivS1, _, Ptr64}}, .OperandMapping: {{Vcc}, {None, VgprPtr64, VgprPtr64}}});
857 // clang-format on
858
859 addRulesForGOpcs(OpcList: {G_BRCOND})
860 .Any(Init: {.Predicate: {UniS1}, .OperandMapping: {{}, {Sgpr32AExtBoolInReg}}})
861 .Any(Init: {.Predicate: {DivS1}, .OperandMapping: {{}, {Vcc}}});
862
863 addRulesForGOpcs(OpcList: {G_BR}).Any(Init: {.Predicate: {_}, .OperandMapping: {{}, {None}}});
864
865 addRulesForGOpcs(OpcList: {G_SELECT}, FastTypes: StandardB)
866 .Any(Init: {.Predicate: {DivS16}, .OperandMapping: {{Vgpr16}, {Vcc, Vgpr16, Vgpr16}}})
867 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{Sgpr16}, {Sgpr32AExtBoolInReg, Sgpr16, Sgpr16}}})
868 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {Vcc, VgprB32, VgprB32}})
869 .Uni(Ty: B32, RuleApplyIDs: {{SgprB32}, {Sgpr32AExtBoolInReg, SgprB32, SgprB32}})
870 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {Vcc, VgprB64, VgprB64}, SplitTo32Select})
871 .Uni(Ty: B64, RuleApplyIDs: {{SgprB64}, {Sgpr32AExtBoolInReg, SgprB64, SgprB64}});
872
873 addRulesForGOpcs(OpcList: {G_ANYEXT})
874 .Any(Init: {.Predicate: {UniS16, S1}, .OperandMapping: {{None}, {None}}}) // should be combined away
875 .Any(Init: {.Predicate: {UniS32, S1}, .OperandMapping: {{None}, {None}}}) // should be combined away
876 .Any(Init: {.Predicate: {UniS64, S1}, .OperandMapping: {{None}, {None}}}) // should be combined away
877 .Any(Init: {.Predicate: {DivS16, S1}, .OperandMapping: {{Vgpr16}, {Vcc}, VccExtToSel}})
878 .Any(Init: {.Predicate: {DivS32, S1}, .OperandMapping: {{Vgpr32}, {Vcc}, VccExtToSel}})
879 .Any(Init: {.Predicate: {DivS64, S1}, .OperandMapping: {{Vgpr64}, {Vcc}, VccExtToSel}})
880 .Any(Init: {.Predicate: {UniS64, S32}, .OperandMapping: {{Sgpr64}, {Sgpr32}, Ext32To64}})
881 .Any(Init: {.Predicate: {DivS64, S32}, .OperandMapping: {{Vgpr64}, {Vgpr32}, Ext32To64}})
882 .Any(Init: {.Predicate: {UniS64, S16}, .OperandMapping: {{Sgpr64}, {Sgpr32AExt}, Ext32To64}})
883 .Any(Init: {.Predicate: {DivS64, S16}, .OperandMapping: {{Vgpr64}, {Vgpr32AExt}, Ext32To64}})
884 .Any(Init: {.Predicate: {UniS32, S16}, .OperandMapping: {{Sgpr32}, {Sgpr16}}})
885 .Any(Init: {.Predicate: {DivS32, S16}, .OperandMapping: {{Vgpr32}, {Vgpr16}}});
886
887 bool Has16bitCmp = ST->has16BitInsts();
888
889 // In global-isel G_TRUNC in-reg is treated as no-op, inst selected into COPY.
890 // It is up to user to deal with truncated bits.
891 // S1, S16, S32 and S64 results are handled with specific rules. Remaining
892 // (result, source) pairs with valid register classes are covered by the
893 // generic UniBRC/DivBRC wildcard rules.
894 addRulesForGOpcs(OpcList: {G_TRUNC})
895 .Any(Init: {.Predicate: {UniS1, UniS16}, .OperandMapping: {{None}, {None}}}) // should be combined away
896 .Any(Init: {.Predicate: {UniS1, UniS32}, .OperandMapping: {{None}, {None}}}) // should be combined away
897 .Any(Init: {.Predicate: {UniS1, UniS64}, .OperandMapping: {{None}, {None}}}) // should be combined away
898 .Any(Init: {.Predicate: {UniS16, S32}, .OperandMapping: {{Sgpr16}, {Sgpr32}}})
899 .Any(Init: {.Predicate: {UniBRC, UniBRC}, .OperandMapping: {{SgprBRC}, {SgprBRC}}})
900 .Any(Init: {.Predicate: {DivBRC, DivBRC}, .OperandMapping: {{VgprBRC}, {VgprBRC}}})
901 .Any(Init: {.Predicate: {UniV2S16, V2S32}, .OperandMapping: {{SgprV2S16}, {SgprV2S32}}})
902 .Any(Init: {.Predicate: {DivV2S16, V2S32}, .OperandMapping: {{VgprV2S16}, {VgprV2S32}}})
903 // This is non-trivial. VgprToVccCopy is done using compare instruction.
904 .Any(Init: {.Predicate: {DivS1, DivS16}, .OperandMapping: {{Vcc}, {Vgpr16}, VgprToVccCopy}}, STPred: Has16bitCmp)
905 .Any(Init: {.Predicate: {DivS1, DivS16}, .OperandMapping: {{Vcc}, {Vgpr32AExt}, VgprToVccCopy}},
906 STPred: !Has16bitCmp)
907 .Any(Init: {.Predicate: {DivS1, DivS32}, .OperandMapping: {{Vcc}, {Vgpr32}, VgprToVccCopy}})
908 .Any(Init: {.Predicate: {DivS1, DivS64}, .OperandMapping: {{Vcc}, {Vgpr64}, VgprToVccCopy}});
909
910 addRulesForGOpcs(OpcList: {G_ZEXT})
911 .Any(Init: {.Predicate: {UniS16, S1}, .OperandMapping: {{Sgpr32Trunc}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
912 .Any(Init: {.Predicate: {UniS32, S1}, .OperandMapping: {{Sgpr32}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
913 .Any(Init: {.Predicate: {UniS64, S1}, .OperandMapping: {{Sgpr64}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
914 .Any(Init: {.Predicate: {DivS16, S1}, .OperandMapping: {{Vgpr16}, {Vcc}, VccExtToSel}})
915 .Any(Init: {.Predicate: {DivS32, S1}, .OperandMapping: {{Vgpr32}, {Vcc}, VccExtToSel}})
916 .Any(Init: {.Predicate: {DivS64, S1}, .OperandMapping: {{Vgpr64}, {Vcc}, VccExtToSel}})
917 .Any(Init: {.Predicate: {UniS64, S32}, .OperandMapping: {{Sgpr64}, {Sgpr32}, Ext32To64}})
918 .Any(Init: {.Predicate: {DivS64, S32}, .OperandMapping: {{Vgpr64}, {Vgpr32}, Ext32To64}})
919 // not extending S16 to S32 is questionable.
920 .Any(Init: {.Predicate: {UniS64, S16}, .OperandMapping: {{Sgpr64}, {Sgpr32ZExt}, Ext32To64}})
921 .Any(Init: {.Predicate: {DivS64, S16}, .OperandMapping: {{Vgpr64}, {Vgpr32ZExt}, Ext32To64}})
922 .Any(Init: {.Predicate: {UniS32, S16}, .OperandMapping: {{Sgpr32}, {Sgpr16}}})
923 .Any(Init: {.Predicate: {DivS32, S16}, .OperandMapping: {{Vgpr32}, {Vgpr16}}});
924
925 addRulesForGOpcs(OpcList: {G_SEXT})
926 .Any(Init: {.Predicate: {UniS16, S1}, .OperandMapping: {{Sgpr32Trunc}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
927 .Any(Init: {.Predicate: {UniS32, S1}, .OperandMapping: {{Sgpr32}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
928 .Any(Init: {.Predicate: {UniS64, S1}, .OperandMapping: {{Sgpr64}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
929 .Any(Init: {.Predicate: {DivS16, S1}, .OperandMapping: {{Vgpr16}, {Vcc}, VccExtToSel}})
930 .Any(Init: {.Predicate: {DivS32, S1}, .OperandMapping: {{Vgpr32}, {Vcc}, VccExtToSel}})
931 .Any(Init: {.Predicate: {DivS64, S1}, .OperandMapping: {{Vgpr64}, {Vcc}, VccExtToSel}})
932 .Any(Init: {.Predicate: {UniS64, S32}, .OperandMapping: {{Sgpr64}, {Sgpr32}, Ext32To64}})
933 .Any(Init: {.Predicate: {DivS64, S32}, .OperandMapping: {{Vgpr64}, {Vgpr32}, Ext32To64}})
934 // not extending S16 to S32 is questionable.
935 .Any(Init: {.Predicate: {UniS64, S16}, .OperandMapping: {{Sgpr64}, {Sgpr32SExt}, Ext32To64}})
936 .Any(Init: {.Predicate: {DivS64, S16}, .OperandMapping: {{Vgpr64}, {Vgpr32SExt}, Ext32To64}})
937 .Any(Init: {.Predicate: {UniS32, S16}, .OperandMapping: {{Sgpr32}, {Sgpr16}}})
938 .Any(Init: {.Predicate: {DivS32, S16}, .OperandMapping: {{Vgpr32}, {Vgpr16}}});
939
940 addRulesForGOpcs(OpcList: {G_SEXT_INREG})
941 .Any(Init: {.Predicate: {UniS32, S32}, .OperandMapping: {{Sgpr32}, {Sgpr32}}})
942 .Any(Init: {.Predicate: {DivS32, S32}, .OperandMapping: {{Vgpr32}, {Vgpr32}}})
943 .Any(Init: {.Predicate: {UniS64, S64}, .OperandMapping: {{Sgpr64}, {Sgpr64}}})
944 .Any(Init: {.Predicate: {DivS64, S64}, .OperandMapping: {{Vgpr64}, {Vgpr64}, SplitTo32SExtInReg}});
945
946 addRulesForGOpcs(OpcList: {G_ASSERT_ZEXT, G_ASSERT_SEXT}, FastTypes: Standard)
947 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Imm}})
948 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Imm}})
949 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {Sgpr64, Imm}})
950 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Imm}});
951
952 addRulesForGOpcs(OpcList: {G_ASSERT_ALIGN}, FastTypes: Standard)
953 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32}})
954 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32}})
955 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {Sgpr64}})
956 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64}})
957 .Any(Init: {.Predicate: {UniPtr32}, .OperandMapping: {{SgprPtr32}, {SgprPtr32}}})
958 .Any(Init: {.Predicate: {DivPtr32}, .OperandMapping: {{VgprPtr32}, {VgprPtr32}}})
959 .Any(Init: {.Predicate: {UniPtr64}, .OperandMapping: {{SgprPtr64}, {SgprPtr64}}})
960 .Any(Init: {.Predicate: {DivPtr64}, .OperandMapping: {{VgprPtr64}, {VgprPtr64}}});
961
962 // Atomic read-modify-write operations: result and value are always VGPR,
963 // pointer varies by address space.
964 addRulesForGOpcs(OpcList: {G_ATOMICRMW_ADD, G_ATOMICRMW_SUB, G_ATOMICRMW_XCHG,
965 G_ATOMICRMW_AND, G_ATOMICRMW_OR, G_ATOMICRMW_XOR,
966 G_ATOMICRMW_MIN, G_ATOMICRMW_MAX, G_ATOMICRMW_UMIN,
967 G_ATOMICRMW_UMAX, G_ATOMICRMW_UINC_WRAP,
968 G_ATOMICRMW_UDEC_WRAP, G_ATOMICRMW_FMIN, G_ATOMICRMW_FMAX})
969 .Any(Init: {.Predicate: {DivS32, P0, S32}, .OperandMapping: {{Vgpr32}, {VgprP0, Vgpr32}}})
970 .Any(Init: {.Predicate: {DivS64, P0, S64}, .OperandMapping: {{Vgpr64}, {VgprP0, Vgpr64}}})
971 .Any(Init: {.Predicate: {DivS32, P1, S32}, .OperandMapping: {{Vgpr32}, {VgprP1, Vgpr32}}})
972 .Any(Init: {.Predicate: {DivS64, P1, S64}, .OperandMapping: {{Vgpr64}, {VgprP1, Vgpr64}}})
973 .Any(Init: {.Predicate: {DivS32, P3, S32}, .OperandMapping: {{Vgpr32}, {VgprP3, Vgpr32}}})
974 .Any(Init: {.Predicate: {DivS64, P3, S64}, .OperandMapping: {{Vgpr64}, {VgprP3, Vgpr64}}});
975
976 addRulesForGOpcs(OpcList: {G_ATOMICRMW_USUB_SAT, G_ATOMICRMW_USUB_COND})
977 .Any(Init: {.Predicate: {DivS32, P0}, .OperandMapping: {{Vgpr32}, {VgprP0, Vgpr32}}})
978 .Any(Init: {.Predicate: {DivS32, P1}, .OperandMapping: {{Vgpr32}, {VgprP1, Vgpr32}}})
979 .Any(Init: {.Predicate: {DivS32, P3}, .OperandMapping: {{Vgpr32}, {VgprP3, Vgpr32}}});
980
981 bool HasAtomicFlatPkAdd16Insts = ST->hasAtomicFlatPkAdd16Insts();
982 bool HasAtomicBufferGlobalPkAddF16Insts =
983 ST->hasAtomicBufferGlobalPkAddF16NoRtnInsts() ||
984 ST->hasAtomicBufferGlobalPkAddF16Insts();
985 bool HasAtomicDsPkAdd16Insts = ST->hasAtomicDsPkAdd16Insts();
986 addRulesForGOpcs(OpcList: {G_ATOMICRMW_FADD})
987 .Any(Init: {.Predicate: {DivS32, P0, S32}, .OperandMapping: {{Vgpr32}, {VgprP0, Vgpr32}}})
988 .Any(Init: {.Predicate: {DivS64, P0, S64}, .OperandMapping: {{Vgpr64}, {VgprP0, Vgpr64}}})
989 .Any(Init: {.Predicate: {DivS32, P1, S32}, .OperandMapping: {{Vgpr32}, {VgprP1, Vgpr32}}})
990 .Any(Init: {.Predicate: {DivS64, P1, S64}, .OperandMapping: {{Vgpr64}, {VgprP1, Vgpr64}}})
991 .Any(Init: {.Predicate: {DivS32, P3, S32}, .OperandMapping: {{Vgpr32}, {VgprP3, Vgpr32}}})
992 .Any(Init: {.Predicate: {DivS64, P3, S64}, .OperandMapping: {{Vgpr64}, {VgprP3, Vgpr64}}})
993 .Any(Init: {.Predicate: {DivV2S16, P0, V2S16}, .OperandMapping: {{VgprV2S16}, {VgprP0, VgprV2S16}}},
994 STPred: HasAtomicFlatPkAdd16Insts)
995 .Any(Init: {.Predicate: {DivV2S16, P1, V2S16}, .OperandMapping: {{VgprV2S16}, {VgprP1, VgprV2S16}}},
996 STPred: HasAtomicBufferGlobalPkAddF16Insts)
997 .Any(Init: {.Predicate: {DivV2S16, P3, V2S16}, .OperandMapping: {{VgprV2S16}, {VgprP3, VgprV2S16}}},
998 STPred: HasAtomicDsPkAdd16Insts);
999
1000 addRulesForGOpcs(OpcList: {G_ATOMIC_CMPXCHG})
1001 .Any(Init: {.Predicate: {DivS32, P2}, .OperandMapping: {{Vgpr32}, {VgprP2, Vgpr32, Vgpr32}}})
1002 .Any(Init: {.Predicate: {DivS64, P2}, .OperandMapping: {{Vgpr64}, {VgprP2, Vgpr64, Vgpr64}}})
1003 .Any(Init: {.Predicate: {DivS32, P3}, .OperandMapping: {{Vgpr32}, {VgprP3, Vgpr32, Vgpr32}}})
1004 .Any(Init: {.Predicate: {DivS64, P3}, .OperandMapping: {{Vgpr64}, {VgprP3, Vgpr64, Vgpr64}}});
1005
1006 addRulesForGOpcs(OpcList: {G_AMDGPU_ATOMIC_CMPXCHG})
1007 .Any(Init: {.Predicate: {DivS32, P0}, .OperandMapping: {{Vgpr32}, {VgprP0, VgprV2S32}}})
1008 .Any(Init: {.Predicate: {DivS32, P1}, .OperandMapping: {{Vgpr32}, {VgprP1, VgprV2S32}}})
1009 .Any(Init: {.Predicate: {DivS64, P0}, .OperandMapping: {{Vgpr64}, {VgprP0, VgprV2S64}}})
1010 .Any(Init: {.Predicate: {DivS64, P1}, .OperandMapping: {{Vgpr64}, {VgprP1, VgprV2S64}}});
1011
1012 addRulesForGOpcs(OpcList: {G_AMDGPU_BUFFER_ATOMIC_CMPSWAP}, FastTypes: Standard)
1013 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32},
1014 {Vgpr32, Vgpr32, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1015 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64},
1016 {Vgpr64, Vgpr64, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
1017
1018 addRulesForGOpcs(OpcList: {G_AMDGPU_BUFFER_ATOMIC_ADD, G_AMDGPU_BUFFER_ATOMIC_AND,
1019 G_AMDGPU_BUFFER_ATOMIC_DEC, G_AMDGPU_BUFFER_ATOMIC_FMAX,
1020 G_AMDGPU_BUFFER_ATOMIC_FMIN, G_AMDGPU_BUFFER_ATOMIC_INC,
1021 G_AMDGPU_BUFFER_ATOMIC_OR, G_AMDGPU_BUFFER_ATOMIC_SMAX,
1022 G_AMDGPU_BUFFER_ATOMIC_SMIN, G_AMDGPU_BUFFER_ATOMIC_SUB,
1023 G_AMDGPU_BUFFER_ATOMIC_SWAP, G_AMDGPU_BUFFER_ATOMIC_UMAX,
1024 G_AMDGPU_BUFFER_ATOMIC_UMIN, G_AMDGPU_BUFFER_ATOMIC_XOR},
1025 FastTypes: Standard)
1026 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1027 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
1028
1029 bool hasSMRDx3 = ST->hasScalarDwordx3Loads();
1030 bool hasSMRDSmall = ST->hasScalarSubwordLoads();
1031 bool usesTrue16 = ST->useRealTrue16Insts();
1032
1033 Predicate isAlign16([](const MachineInstr &MI) -> bool {
1034 return (*MI.memoperands_begin())->getAlign() >= Align(16);
1035 });
1036
1037 Predicate isAlign4([](const MachineInstr &MI) -> bool {
1038 return (*MI.memoperands_begin())->getAlign() >= Align(4);
1039 });
1040
1041 Predicate isAtomicMMO([](const MachineInstr &MI) -> bool {
1042 return (*MI.memoperands_begin())->isAtomic();
1043 });
1044
1045 Predicate isUniMMO([](const MachineInstr &MI) -> bool {
1046 return AMDGPU::isUniformMMO(MMO: *MI.memoperands_begin());
1047 });
1048
1049 Predicate isConst([](const MachineInstr &MI) -> bool {
1050 // Address space in MMO be different then address space on pointer.
1051 const MachineMemOperand *MMO = *MI.memoperands_begin();
1052 const unsigned AS = MMO->getAddrSpace();
1053 return AS == AMDGPUAS::CONSTANT_ADDRESS ||
1054 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT;
1055 });
1056
1057 Predicate isVolatileMMO([](const MachineInstr &MI) -> bool {
1058 return (*MI.memoperands_begin())->isVolatile();
1059 });
1060
1061 Predicate isInvMMO([](const MachineInstr &MI) -> bool {
1062 return (*MI.memoperands_begin())->isInvariant();
1063 });
1064
1065 Predicate isNoClobberMMO([](const MachineInstr &MI) -> bool {
1066 return (*MI.memoperands_begin())->getFlags() & MONoClobber;
1067 });
1068
1069 Predicate isNaturalAligned([](const MachineInstr &MI) -> bool {
1070 const MachineMemOperand *MMO = *MI.memoperands_begin();
1071 return MMO->getAlign() >= Align(MMO->getSize().getValue());
1072 });
1073
1074 Predicate is8Or16BitMMO([](const MachineInstr &MI) -> bool {
1075 const MachineMemOperand *MMO = *MI.memoperands_begin();
1076 const unsigned MemSize = 8 * MMO->getSize().getValue();
1077 return MemSize == 16 || MemSize == 8;
1078 });
1079
1080 Predicate is32BitMMO([](const MachineInstr &MI) -> bool {
1081 const MachineMemOperand *MMO = *MI.memoperands_begin();
1082 return 8 * MMO->getSize().getValue() == 32;
1083 });
1084
1085 auto isUL = !isAtomicMMO && isUniMMO && (isConst || !isVolatileMMO) &&
1086 (isConst || isInvMMO || isNoClobberMMO);
1087
1088 // clang-format off
1089 // TODO: S32Dst, 16-bit any-extending load should not appear on True16 targets
1090 addRulesForGOpcs(OpcList: {G_LOAD})
1091 // flat, addrspace(0), never uniform - flat_load
1092 .Any(Init: {.Predicate: {DivS16, P0}, .OperandMapping: {{Vgpr16}, {VgprP0}}}, STPred: usesTrue16)
1093 .Any(Init: {.Predicate: {DivB32, P0}, .OperandMapping: {{VgprB32}, {VgprP0}}}) // 32-bit load, 8-bit and 16-bit any-extending load
1094 .Any(Init: {.Predicate: {DivB64, P0}, .OperandMapping: {{VgprB64}, {VgprP0}}})
1095 .Any(Init: {.Predicate: {DivB96, P0}, .OperandMapping: {{VgprB96}, {VgprP0}}})
1096 .Any(Init: {.Predicate: {DivB128, P0}, .OperandMapping: {{VgprB128}, {VgprP0}}})
1097
1098 // global, addrspace(1)
1099 // divergent - global_load
1100 .Any(Init: {.Predicate: {DivS16, P1}, .OperandMapping: {{Vgpr16}, {VgprP1}}}, STPred: usesTrue16)
1101 .Any(Init: {.Predicate: {DivB32, P1}, .OperandMapping: {{VgprB32}, {VgprP1}}}) //32-bit load, 8-bit and 16-bit any-extending load
1102 .Any(Init: {.Predicate: {DivB64, P1}, .OperandMapping: {{VgprB64}, {VgprP1}}})
1103 .Any(Init: {.Predicate: {DivB96, P1}, .OperandMapping: {{VgprB96}, {VgprP1}}})
1104 .Any(Init: {.Predicate: {DivB128, P1}, .OperandMapping: {{VgprB128}, {VgprP1}}})
1105 .Any(Init: {.Predicate: {DivB256, P1}, .OperandMapping: {{VgprB256}, {VgprP1}, SplitLoad}})
1106 .Any(Init: {.Predicate: {DivB512, P1}, .OperandMapping: {{VgprB512}, {VgprP1}, SplitLoad}})
1107
1108 // uniform - s_load
1109 .Any(Init: {.Predicate: {{UniS16, P1}, isNaturalAligned && isUL}, .OperandMapping: {{Sgpr32Trunc}, {SgprP1}}}, STPred: usesTrue16 && hasSMRDSmall) // s16 load
1110 .Any(Init: {.Predicate: {{UniS16, P1}, isAlign4 && isUL}, .OperandMapping: {{Sgpr32Trunc}, {SgprP1}, WidenMMOToS32}}, STPred: usesTrue16 && !hasSMRDSmall) // s16 load to 32-bit load
1111 .Any(Init: {.Predicate: {{UniB32, P1}, isNaturalAligned && isUL}, .OperandMapping: {{SgprB32}, {SgprP1}}}, STPred: hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load
1112 // TODO: SplitLoad when !isNaturalAligned && isUL and target hasSMRDSmall
1113 .Any(Init: {.Predicate: {{UniB32, P1}, is8Or16BitMMO && isAlign4 && isUL}, .OperandMapping: {{SgprB32}, {SgprP1}, WidenMMOToS32}}, STPred: !hasSMRDSmall) //8-bit and 16-bit any-extending load to 32-bit load
1114 .Any(Init: {.Predicate: {{UniB32, P1}, is32BitMMO && isAlign4 && isUL}, .OperandMapping: {{SgprB32}, {SgprP1}}}) //32-bit load
1115 .Any(Init: {.Predicate: {{UniB64, P1}, isAlign4 && isUL}, .OperandMapping: {{SgprB64}, {SgprP1}}})
1116 .Any(Init: {.Predicate: {{UniB96, P1}, isAlign16 && isUL}, .OperandMapping: {{SgprB96}, {SgprP1}, WidenLoad}}, STPred: !hasSMRDx3)
1117 .Any(Init: {.Predicate: {{UniB96, P1}, isAlign4 && !isAlign16 && isUL}, .OperandMapping: {{SgprB96}, {SgprP1}, SplitLoad}}, STPred: !hasSMRDx3)
1118 .Any(Init: {.Predicate: {{UniB96, P1}, isAlign4 && isUL}, .OperandMapping: {{SgprB96}, {SgprP1}}}, STPred: hasSMRDx3)
1119 .Any(Init: {.Predicate: {{UniB128, P1}, isAlign4 && isUL}, .OperandMapping: {{SgprB128}, {SgprP1}}})
1120 .Any(Init: {.Predicate: {{UniB256, P1}, isAlign4 && isUL}, .OperandMapping: {{SgprB256}, {SgprP1}}})
1121 .Any(Init: {.Predicate: {{UniB512, P1}, isAlign4 && isUL}, .OperandMapping: {{SgprB512}, {SgprP1}}})
1122
1123 // Uniform via global or buffer load, for example volatile or non-aligned
1124 // uniform load. Not using standard {{UniInVgprTy}, {VgprP1}} since it is
1125 // selected as global_load, use SgprP1 for pointer instead to match
1126 // patterns without flat-for-global, default for GFX7 and older.
1127 // -> +flat-for-global + {{UniInVgprTy}, {SgprP1}} - global_load
1128 // -> -flat-for-global + {{UniInVgprTy}, {SgprP1}} - buffer_load
1129 .Any(Init: {.Predicate: {{UniS16, P1}, !isNaturalAligned || !isUL}, .OperandMapping: {{UniInVgprS16}, {SgprP1}}}, STPred: usesTrue16 && hasSMRDSmall) // s16 load
1130 .Any(Init: {.Predicate: {{UniS16, P1}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprS16}, {SgprP1}}}, STPred: usesTrue16 && !hasSMRDSmall) // s16 load
1131 .Any(Init: {.Predicate: {{UniB32, P1}, !isNaturalAligned || !isUL}, .OperandMapping: {{UniInVgprB32}, {SgprP1}}}, STPred: hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load
1132 .Any(Init: {.Predicate: {{UniB32, P1}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB32}, {SgprP1}}}, STPred: !hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load
1133 .Any(Init: {.Predicate: {{UniB64, P1}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB64}, {SgprP1}}})
1134 .Any(Init: {.Predicate: {{UniB96, P1}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB96}, {SgprP1}}})
1135 .Any(Init: {.Predicate: {{UniB128, P1}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB128}, {SgprP1}}})
1136 .Any(Init: {.Predicate: {{UniB256, P1}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB256}, {SgprP1}, SplitLoad}})
1137 .Any(Init: {.Predicate: {{UniB512, P1}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB512}, {SgprP1}, SplitLoad}})
1138
1139 // local, addrspace(3) - ds_load
1140 .Any(Init: {.Predicate: {DivS16, P3}, .OperandMapping: {{Vgpr16}, {VgprP3}}}, STPred: usesTrue16)
1141 .Any(Init: {.Predicate: {DivB32, P3}, .OperandMapping: {{VgprB32}, {VgprP3}}}) // 32-bit load, 8-bit and 16-bit any-extending load
1142 .Any(Init: {.Predicate: {DivB64, P3}, .OperandMapping: {{VgprB64}, {VgprP3}}})
1143 .Any(Init: {.Predicate: {DivB96, P3}, .OperandMapping: {{VgprB96}, {VgprP3}}})
1144 .Any(Init: {.Predicate: {DivB128, P3}, .OperandMapping: {{VgprB128}, {VgprP3}}})
1145
1146 .Any(Init: {.Predicate: {UniS16, P3}, .OperandMapping: {{UniInVgprS16}, {SgprP3}}}, STPred: usesTrue16) // 16-bit load
1147 .Any(Init: {.Predicate: {UniB32, P3}, .OperandMapping: {{UniInVgprB32}, {VgprP3}}}) // 32-bit load, 8-bit and 16-bit any-extending load
1148 .Any(Init: {.Predicate: {UniB64, P3}, .OperandMapping: {{UniInVgprB64}, {VgprP3}}})
1149 .Any(Init: {.Predicate: {UniB96, P3}, .OperandMapping: {{UniInVgprB96}, {VgprP3}}})
1150 .Any(Init: {.Predicate: {UniB128, P3}, .OperandMapping: {{UniInVgprB128}, {VgprP3}}})
1151
1152 // constant, addrspace(4)
1153 // divergent - global_load
1154 .Any(Init: {.Predicate: {DivS16, P4}, .OperandMapping: {{Vgpr16}, {VgprP4}}}, STPred: usesTrue16)
1155 .Any(Init: {.Predicate: {DivB32, P4}, .OperandMapping: {{VgprB32}, {VgprP4}}}) //32-bit load, 8-bit and 16-bit any-extending load
1156 .Any(Init: {.Predicate: {DivB64, P4}, .OperandMapping: {{VgprB64}, {VgprP4}}})
1157 .Any(Init: {.Predicate: {DivB96, P4}, .OperandMapping: {{VgprB96}, {VgprP4}}})
1158 .Any(Init: {.Predicate: {DivB128, P4}, .OperandMapping: {{VgprB128}, {VgprP4}}})
1159 .Any(Init: {.Predicate: {DivB256, P4}, .OperandMapping: {{VgprB256}, {VgprP4}, SplitLoad}})
1160 .Any(Init: {.Predicate: {DivB512, P4}, .OperandMapping: {{VgprB512}, {VgprP4}, SplitLoad}})
1161
1162 // uniform - s_load
1163 .Any(Init: {.Predicate: {{UniS16, P4}, isNaturalAligned && isUL}, .OperandMapping: {{Sgpr32Trunc}, {SgprP4}}}, STPred: usesTrue16 && hasSMRDSmall) // s16 load
1164 .Any(Init: {.Predicate: {{UniS16, P4}, isAlign4 && isUL}, .OperandMapping: {{Sgpr32Trunc}, {SgprP4}, WidenMMOToS32}}, STPred: usesTrue16 && !hasSMRDSmall) // s16 load to 32-bit load
1165 .Any(Init: {.Predicate: {{UniB32, P4}, isNaturalAligned && isUL}, .OperandMapping: {{SgprB32}, {SgprP4}}}, STPred: hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load
1166 .Any(Init: {.Predicate: {{UniB32, P4}, is8Or16BitMMO && isAlign4 && isUL}, .OperandMapping: {{SgprB32}, {SgprP4}, WidenMMOToS32}}, STPred: !hasSMRDSmall) //8-bit and 16-bit any-extending load to 32-bit load
1167 .Any(Init: {.Predicate: {{UniB32, P4}, is32BitMMO && isAlign4 && isUL}, .OperandMapping: {{SgprB32}, {SgprP4}}}) //32-bit load
1168 .Any(Init: {.Predicate: {{UniB64, P4}, isAlign4 && isUL}, .OperandMapping: {{SgprB64}, {SgprP4}}})
1169 .Any(Init: {.Predicate: {{UniB96, P4}, isAlign16 && isUL}, .OperandMapping: {{SgprB96}, {SgprP4}, WidenLoad}}, STPred: !hasSMRDx3)
1170 .Any(Init: {.Predicate: {{UniB96, P4}, isAlign4 && !isAlign16 && isUL}, .OperandMapping: {{SgprB96}, {SgprP4}, SplitLoad}}, STPred: !hasSMRDx3)
1171 .Any(Init: {.Predicate: {{UniB96, P4}, isAlign4 && isUL}, .OperandMapping: {{SgprB96}, {SgprP4}}}, STPred: hasSMRDx3)
1172 .Any(Init: {.Predicate: {{UniB128, P4}, isAlign4 && isUL}, .OperandMapping: {{SgprB128}, {SgprP4}}})
1173 .Any(Init: {.Predicate: {{UniB256, P4}, isAlign4 && isUL}, .OperandMapping: {{SgprB256}, {SgprP4}}})
1174 .Any(Init: {.Predicate: {{UniB512, P4}, isAlign4 && isUL}, .OperandMapping: {{SgprB512}, {SgprP4}}})
1175
1176 // uniform in vgpr - global_load or buffer_load
1177 .Any(Init: {.Predicate: {{UniS16, P4}, !isNaturalAligned || !isUL}, .OperandMapping: {{UniInVgprS16}, {SgprP4}}}, STPred: usesTrue16 && hasSMRDSmall) // s16 load
1178 .Any(Init: {.Predicate: {{UniS16, P4}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprS16}, {SgprP4}}}, STPred: usesTrue16 && !hasSMRDSmall) // s16 load
1179 .Any(Init: {.Predicate: {{UniB32, P4}, !isNaturalAligned || !isUL}, .OperandMapping: {{UniInVgprB32}, {SgprP4}}}, STPred: hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load
1180 .Any(Init: {.Predicate: {{UniB32, P4}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB32}, {SgprP4}}}, STPred: !hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load
1181 .Any(Init: {.Predicate: {{UniB64, P4}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB64}, {SgprP4}}})
1182 .Any(Init: {.Predicate: {{UniB96, P4}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB96}, {SgprP4}}})
1183 .Any(Init: {.Predicate: {{UniB128, P4}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB128}, {SgprP4}}})
1184 .Any(Init: {.Predicate: {{UniB256, P4}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB256}, {SgprP4}, SplitLoad}})
1185 .Any(Init: {.Predicate: {{UniB512, P4}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB512}, {SgprP4}, SplitLoad}})
1186
1187 // private, addrspace(5), never uniform - scratch_load
1188 .Any(Init: {.Predicate: {DivS16, P5}, .OperandMapping: {{Vgpr16}, {VgprP5}}}, STPred: usesTrue16)
1189 .Any(Init: {.Predicate: {DivB32, P5}, .OperandMapping: {{VgprB32}, {VgprP5}}}) // 32-bit load, 8-bit and 16-bit any-extending load
1190 .Any(Init: {.Predicate: {DivB64, P5}, .OperandMapping: {{VgprB64}, {VgprP5}}})
1191 .Any(Init: {.Predicate: {DivB96, P5}, .OperandMapping: {{VgprB96}, {VgprP5}}})
1192 .Any(Init: {.Predicate: {DivB128, P5}, .OperandMapping: {{VgprB128}, {VgprP5}}})
1193
1194 .Any(Init: {.Predicate: {DivS32, Ptr128}, .OperandMapping: {{Vgpr32}, {VgprPtr128}}});
1195
1196
1197 addRulesForGOpcs(OpcList: {G_ZEXTLOAD, G_SEXTLOAD}) // i8 and i16 zeroextending loads
1198 .Any(Init: {.Predicate: {DivS32, P0}, .OperandMapping: {{Vgpr32}, {VgprP0}}})
1199 .Any(Init: {.Predicate: {DivS16, P0}, .OperandMapping: {{Vgpr16}, {VgprP0}}}, STPred: usesTrue16)
1200
1201 .Any(Init: {.Predicate: {DivS32, P1}, .OperandMapping: {{Vgpr32}, {VgprP1}}})
1202 .Any(Init: {.Predicate: {DivS16, P1}, .OperandMapping: {{Vgpr16}, {VgprP1}}}, STPred: usesTrue16)
1203 .Any(Init: {.Predicate: {{UniS16, P1}, isUL}, .OperandMapping: {{Sgpr32Trunc}, {SgprP1}}}, STPred: usesTrue16 && hasSMRDSmall)
1204 .Any(Init: {.Predicate: {{UniS16, P1}, !isUL}, .OperandMapping: {{UniInVgprS16}, {SgprP1}}}, STPred: usesTrue16 && hasSMRDSmall)
1205 .Any(Init: {.Predicate: {UniS16, P1}, .OperandMapping: {{UniInVgprS16}, {SgprP1}}}, STPred: usesTrue16 && !hasSMRDSmall)
1206 .Any(Init: {.Predicate: {{UniS32, P1}, isAlign4 && isUL}, .OperandMapping: {{Sgpr32}, {SgprP1}, WidenMMOToS32}}, STPred: !hasSMRDSmall)
1207 .Any(Init: {.Predicate: {{UniS32, P1}, isNaturalAligned && isUL}, .OperandMapping: {{Sgpr32}, {SgprP1}}}, STPred: hasSMRDSmall)
1208 .Any(Init: {.Predicate: {{UniS32, P1}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprS32}, {SgprP1}}}, STPred: !hasSMRDSmall)
1209 .Any(Init: {.Predicate: {{UniS32, P1}, !isNaturalAligned || !isUL}, .OperandMapping: {{UniInVgprS32}, {SgprP1}}}, STPred: hasSMRDSmall)
1210
1211 .Any(Init: {.Predicate: {DivS32, P3}, .OperandMapping: {{Vgpr32}, {VgprP3}}})
1212 .Any(Init: {.Predicate: {DivS16, P3}, .OperandMapping: {{Vgpr16}, {VgprP3}}}, STPred: usesTrue16)
1213 .Any(Init: {.Predicate: {UniS16, P3}, .OperandMapping: {{UniInVgprS16}, {SgprP3}}}, STPred: usesTrue16)
1214 .Any(Init: {.Predicate: {UniS32, P3}, .OperandMapping: {{UniInVgprS32}, {VgprP3}}})
1215
1216 .Any(Init: {.Predicate: {DivS32, P4}, .OperandMapping: {{Vgpr32}, {VgprP4}}})
1217 .Any(Init: {.Predicate: {DivS16, P4}, .OperandMapping: {{Vgpr16}, {VgprP4}}}, STPred: usesTrue16)
1218 .Any(Init: {.Predicate: {{UniS16, P4}, isUL}, .OperandMapping: {{Sgpr32Trunc}, {SgprP4}}}, STPred: usesTrue16 && hasSMRDSmall)
1219 .Any(Init: {.Predicate: {UniS16, P4}, .OperandMapping: {{UniInVgprS16}, {SgprP4}}}, STPred: usesTrue16 && !hasSMRDSmall)
1220 .Any(Init: {.Predicate: {{UniS32, P4}, isAlign4 && isUL}, .OperandMapping: {{Sgpr32}, {SgprP4}, WidenMMOToS32}}, STPred: !hasSMRDSmall)
1221 .Any(Init: {.Predicate: {{UniS32, P4}, isNaturalAligned && isUL}, .OperandMapping: {{Sgpr32}, {SgprP4}}}, STPred: hasSMRDSmall)
1222 .Any(Init: {.Predicate: {{UniS32, P4}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprS32}, {SgprP4}}}, STPred: !hasSMRDSmall)
1223 .Any(Init: {.Predicate: {{UniS32, P4}, !isNaturalAligned || !isUL}, .OperandMapping: {{UniInVgprS32}, {SgprP4}}}, STPred: hasSMRDSmall)
1224
1225 .Any(Init: {.Predicate: {DivS32, P5}, .OperandMapping: {{Vgpr32}, {VgprP5}}})
1226 .Any(Init: {.Predicate: {DivS16, P5}, .OperandMapping: {{Vgpr16}, {VgprP5}}}, STPred: usesTrue16);
1227
1228 addRulesForGOpcs(OpcList: {G_STORE})
1229 // addrspace(0)
1230 .Any(Init: {.Predicate: {S16, P0}, .OperandMapping: {{}, {Vgpr16, VgprP0}}}, STPred: usesTrue16) // 16-bit store
1231 .Any(Init: {.Predicate: {B32, P0}, .OperandMapping: {{}, {VgprB32, VgprP0}}}) // 32-bit store, 8-bit and 16-bit truncating store
1232 .Any(Init: {.Predicate: {B64, P0}, .OperandMapping: {{}, {VgprB64, VgprP0}}})
1233 .Any(Init: {.Predicate: {B96, P0}, .OperandMapping: {{}, {VgprB96, VgprP0}}})
1234 .Any(Init: {.Predicate: {B128, P0}, .OperandMapping: {{}, {VgprB128, VgprP0}}})
1235
1236 // addrspace(1), there are no stores to addrspace(4)
1237 // For targets:
1238 // - with "+flat-for-global" - global_store
1239 // - without(-flat-for-global) - buffer_store addr64
1240 .Any(Init: {.Predicate: {S16, DivP1}, .OperandMapping: {{}, {Vgpr16, VgprP1}}}, STPred: usesTrue16) // 16-bit store
1241 .Any(Init: {.Predicate: {B32, DivP1}, .OperandMapping: {{}, {VgprB32, VgprP1}}}) // 32-bit store, 8-bit and 16-bit truncating store
1242 .Any(Init: {.Predicate: {B64, DivP1}, .OperandMapping: {{}, {VgprB64, VgprP1}}})
1243 .Any(Init: {.Predicate: {B96, DivP1}, .OperandMapping: {{}, {VgprB96, VgprP1}}})
1244 .Any(Init: {.Predicate: {B128, DivP1}, .OperandMapping: {{}, {VgprB128, VgprP1}}})
1245
1246 // For UniP1, use sgpr ptr to match flat-for-global patterns. Targets:
1247 // - with "+flat-for-global" - global_store for both sgpr and vgpr ptr
1248 // - without(-flat-for-global) - need sgpr ptr to select buffer_store
1249 .Any(Init: {.Predicate: {S16, UniP1}, .OperandMapping: {{}, {Vgpr16, SgprP1}}}, STPred: usesTrue16) // 16-bit store
1250 .Any(Init: {.Predicate: {B32, UniP1}, .OperandMapping: {{}, {VgprB32, SgprP1}}}) // 32-bit store, 8-bit and 16-bit truncating store
1251 .Any(Init: {.Predicate: {B64, UniP1}, .OperandMapping: {{}, {VgprB64, SgprP1}}})
1252 .Any(Init: {.Predicate: {B96, UniP1}, .OperandMapping: {{}, {VgprB96, SgprP1}}})
1253 .Any(Init: {.Predicate: {B128, UniP1}, .OperandMapping: {{}, {VgprB128, SgprP1}}})
1254
1255 // addrspace(3) and addrspace(5)
1256 .Any(Init: {.Predicate: {S16, Ptr32}, .OperandMapping: {{}, {Vgpr16, VgprPtr32}}}, STPred: usesTrue16) // 16-bit store
1257 .Any(Init: {.Predicate: {B32, Ptr32}, .OperandMapping: {{}, {VgprB32, VgprPtr32}}}) // 32-bit store, 8-bit and 16-bit truncating store
1258 .Any(Init: {.Predicate: {B64, Ptr32}, .OperandMapping: {{}, {VgprB64, VgprPtr32}}})
1259 .Any(Init: {.Predicate: {B96, Ptr32}, .OperandMapping: {{}, {VgprB96, VgprPtr32}}})
1260 .Any(Init: {.Predicate: {B128, Ptr32}, .OperandMapping: {{}, {VgprB128, VgprPtr32}}});
1261
1262 // clang-format on
1263
1264 addRulesForGOpcs(OpcList: {G_AMDGPU_BUFFER_LOAD, G_AMDGPU_BUFFER_LOAD_FORMAT,
1265 G_AMDGPU_TBUFFER_LOAD_FORMAT},
1266 FastTypes: StandardB)
1267 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1268 .Uni(Ty: B32, RuleApplyIDs: {{UniInVgprB32}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1269 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1270 .Uni(Ty: B64, RuleApplyIDs: {{UniInVgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1271 .Div(Ty: B96, RuleApplyIDs: {{VgprB96}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1272 .Uni(Ty: B96, RuleApplyIDs: {{UniInVgprB96}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1273 .Div(Ty: B128, RuleApplyIDs: {{VgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1274 .Uni(Ty: B128, RuleApplyIDs: {{UniInVgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
1275
1276 addRulesForGOpcs(OpcList: {G_AMDGPU_BUFFER_LOAD_USHORT, G_AMDGPU_BUFFER_LOAD_UBYTE,
1277 G_AMDGPU_BUFFER_LOAD_SSHORT, G_AMDGPU_BUFFER_LOAD_SBYTE},
1278 FastTypes: StandardB)
1279 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1280 .Uni(Ty: B32, RuleApplyIDs: {{UniInVgprB32}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
1281
1282 addRulesForGOpcs(
1283 OpcList: {G_AMDGPU_BUFFER_LOAD_UBYTE_TFE, G_AMDGPU_BUFFER_LOAD_USHORT_TFE},
1284 FastTypes: StandardB)
1285 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1286 .Uni(Ty: B64, RuleApplyIDs: {{UniInVgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
1287
1288 addRulesForGOpcs(OpcList: {G_AMDGPU_BUFFER_LOAD_TFE, G_AMDGPU_BUFFER_LOAD_FORMAT_TFE,
1289 G_AMDGPU_BUFFER_LOAD_FORMAT_D16_TFE},
1290 FastTypes: StandardB)
1291 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1292 .Uni(Ty: B64, RuleApplyIDs: {{UniInVgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1293 .Div(Ty: B96, RuleApplyIDs: {{VgprB96}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1294 .Uni(Ty: B96, RuleApplyIDs: {{UniInVgprB96}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1295 .Div(Ty: B128, RuleApplyIDs: {{VgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1296 .Uni(Ty: B128, RuleApplyIDs: {{UniInVgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1297 .Any(Init: {.Predicate: {DivB160}, .OperandMapping: {{VgprB160}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}})
1298 .Any(Init: {.Predicate: {UniB160},
1299 .OperandMapping: {{UniInVgprB160}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}});
1300
1301 addRulesForGOpcs(
1302 OpcList: {G_AMDGPU_BUFFER_LOAD_FORMAT_D16, G_AMDGPU_TBUFFER_LOAD_FORMAT_D16},
1303 FastTypes: StandardB)
1304 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1305 .Uni(Ty: B32, RuleApplyIDs: {{UniInVgprB32}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1306 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1307 .Uni(Ty: B64, RuleApplyIDs: {{UniInVgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1308 .Div(Ty: B128, RuleApplyIDs: {{VgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1309 .Uni(Ty: B128, RuleApplyIDs: {{UniInVgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
1310
1311 addRulesForGOpcs(OpcList: {G_AMDGPU_S_BUFFER_LOAD})
1312 // waterfall expansion is part of S_BUF_to_BUF
1313 .Any(Init: {.Predicate: {UniB32}, .OperandMapping: {{SgprB32}, {SgprV4S32, Sgpr32}}})
1314 .Any(Init: {.Predicate: {DivB32, UniV4S32, DivB32},
1315 .OperandMapping: {{VgprB32}, {SgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1316 .Any(Init: {.Predicate: {DivB32, DivV4S32, UniB32},
1317 .OperandMapping: {{VgprB32}, {VgprV4S32, Sgpr32}, S_BUF_to_BUF}})
1318 .Any(Init: {.Predicate: {DivB32, DivV4S32, DivB32},
1319 .OperandMapping: {{VgprB32}, {VgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1320
1321 .Any(Init: {.Predicate: {UniB64}, .OperandMapping: {{SgprB64}, {SgprV4S32, Sgpr32}}})
1322 .Any(Init: {.Predicate: {DivB64, UniV4S32, DivB32},
1323 .OperandMapping: {{VgprB64}, {SgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1324 .Any(Init: {.Predicate: {DivB64, DivV4S32, UniB32},
1325 .OperandMapping: {{VgprB64}, {VgprV4S32, Sgpr32}, S_BUF_to_BUF}})
1326 .Any(Init: {.Predicate: {DivB64, DivV4S32, DivB32},
1327 .OperandMapping: {{VgprB64}, {VgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1328
1329 .Any(Init: {.Predicate: {UniB96}, .OperandMapping: {{SgprB96}, {SgprV4S32, Sgpr32}}})
1330 .Any(Init: {.Predicate: {DivB96, UniV4S32, DivB32},
1331 .OperandMapping: {{VgprB96}, {SgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1332 .Any(Init: {.Predicate: {DivB96, DivV4S32, UniB32},
1333 .OperandMapping: {{VgprB96}, {VgprV4S32, Sgpr32}, S_BUF_to_BUF}})
1334 .Any(Init: {.Predicate: {DivB96, DivV4S32, DivB32},
1335 .OperandMapping: {{VgprB96}, {VgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1336
1337 .Any(Init: {.Predicate: {UniB128}, .OperandMapping: {{SgprB128}, {SgprV4S32, Sgpr32}}})
1338 .Any(Init: {.Predicate: {DivB128, UniV4S32, DivB32},
1339 .OperandMapping: {{VgprB128}, {SgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1340 .Any(Init: {.Predicate: {DivB128, DivV4S32, UniB32},
1341 .OperandMapping: {{VgprB128}, {VgprV4S32, Sgpr32}, S_BUF_to_BUF}})
1342 .Any(Init: {.Predicate: {DivB128, DivV4S32, DivB32},
1343 .OperandMapping: {{VgprB128}, {VgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1344
1345 .Any(Init: {.Predicate: {UniB256}, .OperandMapping: {{SgprB256}, {SgprV4S32, Sgpr32}}})
1346 .Any(Init: {.Predicate: {DivB256, UniV4S32, DivB32},
1347 .OperandMapping: {{VgprB256}, {SgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1348 .Any(Init: {.Predicate: {DivB256, DivV4S32, UniB32},
1349 .OperandMapping: {{VgprB256}, {VgprV4S32, Sgpr32}, S_BUF_to_BUF}})
1350 .Any(Init: {.Predicate: {DivB256, DivV4S32, DivB32},
1351 .OperandMapping: {{VgprB256}, {VgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1352
1353 .Any(Init: {.Predicate: {UniB512}, .OperandMapping: {{SgprB512}, {SgprV4S32, Sgpr32}}})
1354 .Any(Init: {.Predicate: {DivB512, UniV4S32, DivB32},
1355 .OperandMapping: {{VgprB512}, {SgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1356 .Any(Init: {.Predicate: {DivB512, DivV4S32, UniB32},
1357 .OperandMapping: {{VgprB512}, {VgprV4S32, Sgpr32}, S_BUF_to_BUF}})
1358 .Any(Init: {.Predicate: {DivB512, DivV4S32, DivB32},
1359 .OperandMapping: {{VgprB512}, {VgprV4S32, Vgpr32}, S_BUF_to_BUF}});
1360
1361 addRulesForGOpcs(OpcList: {G_AMDGPU_S_BUFFER_LOAD_SBYTE, G_AMDGPU_S_BUFFER_LOAD_UBYTE,
1362 G_AMDGPU_S_BUFFER_LOAD_SSHORT,
1363 G_AMDGPU_S_BUFFER_LOAD_USHORT})
1364 .Any(Init: {.Predicate: {UniS32, UniV4S32, UniS32}, .OperandMapping: {{Sgpr32}, {SgprV4S32, Sgpr32}}})
1365 .Any(Init: {.Predicate: {DivS32, UniV4S32, DivS32},
1366 .OperandMapping: {{Vgpr32}, {SgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1367 .Any(Init: {.Predicate: {DivS32, DivV4S32, UniS32},
1368 .OperandMapping: {{Vgpr32}, {VgprV4S32, Sgpr32}, S_BUF_to_BUF}})
1369 .Any(Init: {.Predicate: {DivS32, DivV4S32, DivS32},
1370 .OperandMapping: {{Vgpr32}, {VgprV4S32, Vgpr32}, S_BUF_to_BUF}});
1371
1372 addRulesForGOpcs(OpcList: {G_AMDGPU_BUFFER_STORE, G_AMDGPU_BUFFER_STORE_BYTE,
1373 G_AMDGPU_BUFFER_STORE_SHORT, G_AMDGPU_BUFFER_STORE_FORMAT,
1374 G_AMDGPU_BUFFER_STORE_FORMAT_D16,
1375 G_AMDGPU_TBUFFER_STORE_FORMAT,
1376 G_AMDGPU_TBUFFER_STORE_FORMAT_D16})
1377 .Any(Init: {.Predicate: {B32}, .OperandMapping: {{}, {VgprB32, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}})
1378 .Any(Init: {.Predicate: {B64}, .OperandMapping: {{}, {VgprB64, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}})
1379 .Any(Init: {.Predicate: {B96}, .OperandMapping: {{}, {VgprB96, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}})
1380 .Any(Init: {.Predicate: {B128}, .OperandMapping: {{}, {VgprB128, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}});
1381
1382 // Buffer atomics: resource descriptor + scalar offset are SGPR, data and
1383 // address components are VGPR.
1384 //
1385 // Operand order (SIInstructions.td BufferAtomicGenericInstruction):
1386 // dst = op vdata, rsrc, vindex, voffset, soffset, offset_imm, cachepolicy,
1387 // idxen_imm
1388 addRulesForGOpcs(OpcList: {G_AMDGPU_BUFFER_ATOMIC_FADD})
1389 .Any(Init: {.Predicate: {S32, S32, V4S32, S32, S32, S32},
1390 .OperandMapping: {{Vgpr32}, {Vgpr32, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}})
1391 .Any(Init: {.Predicate: {S64, S64, V4S32, S32, S32, S32},
1392 .OperandMapping: {{Vgpr64}, {Vgpr64, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}})
1393 .Any(Init: {.Predicate: {V2S16, V2S16, V4S32, S32, S32, S32},
1394 .OperandMapping: {{VgprV2S16},
1395 {VgprV2S16, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}});
1396
1397 addRulesForGOpcs(OpcList: {G_AMDGPU_REG_LOAD}).Any(Init: {.Predicate: {BRC}, .OperandMapping: {{VgprBRC}, {Sgpr32_WF}}});
1398
1399 addRulesForGOpcs(OpcList: {G_AMDGPU_REG_STORE})
1400 .Any(Init: {.Predicate: {BRC}, .OperandMapping: {{}, {VgprBRC, Sgpr32_WF}}});
1401
1402 addRulesForGOpcs(OpcList: {G_PTR_ADD})
1403 .Any(Init: {.Predicate: {UniPtr32}, .OperandMapping: {{SgprPtr32}, {SgprPtr32, Sgpr32}}})
1404 .Any(Init: {.Predicate: {DivPtr32}, .OperandMapping: {{VgprPtr32}, {VgprPtr32, Vgpr32}}})
1405 .Any(Init: {.Predicate: {UniPtr64}, .OperandMapping: {{SgprPtr64}, {SgprPtr64, Sgpr64}}})
1406 .Any(Init: {.Predicate: {DivPtr64}, .OperandMapping: {{VgprPtr64}, {VgprPtr64, Vgpr64}}});
1407
1408 addRulesForGOpcs(OpcList: {G_INTTOPTR})
1409 .Any(Init: {.Predicate: {UniPtr32}, .OperandMapping: {{SgprPtr32}, {Sgpr32}}})
1410 .Any(Init: {.Predicate: {DivPtr32}, .OperandMapping: {{VgprPtr32}, {Vgpr32}}})
1411 .Any(Init: {.Predicate: {UniPtr64}, .OperandMapping: {{SgprPtr64}, {Sgpr64}}})
1412 .Any(Init: {.Predicate: {DivPtr64}, .OperandMapping: {{VgprPtr64}, {Vgpr64}}})
1413 .Any(Init: {.Predicate: {UniPtr128}, .OperandMapping: {{SgprPtr128}, {Sgpr128}}})
1414 .Any(Init: {.Predicate: {DivPtr128}, .OperandMapping: {{VgprPtr128}, {Vgpr128}}});
1415
1416 addRulesForGOpcs(OpcList: {G_PTRTOINT})
1417 .Any(Init: {.Predicate: {UniS32}, .OperandMapping: {{Sgpr32}, {SgprPtr32}}})
1418 .Any(Init: {.Predicate: {DivS32}, .OperandMapping: {{Vgpr32}, {VgprPtr32}}})
1419 .Any(Init: {.Predicate: {UniS64}, .OperandMapping: {{Sgpr64}, {SgprPtr64}}})
1420 .Any(Init: {.Predicate: {DivS64}, .OperandMapping: {{Vgpr64}, {VgprPtr64}}})
1421 .Any(Init: {.Predicate: {UniS128}, .OperandMapping: {{Sgpr128}, {SgprPtr128}}})
1422 .Any(Init: {.Predicate: {DivS128}, .OperandMapping: {{Vgpr128}, {VgprPtr128}}});
1423
1424 // FIXME: Update llvm/test/CodeGen/AMDGPU/ptrmask.ll to use GlobalISel.
1425 // Currently crashes on P8 (buffer resource) tests due to legalizer issue.
1426 addRulesForGOpcs(OpcList: {G_PTRMASK})
1427 .Any(Init: {.Predicate: {UniP1}, .OperandMapping: {{SgprP1}, {SgprP1, Sgpr64}}})
1428 .Any(Init: {.Predicate: {DivP1}, .OperandMapping: {{VgprP1}, {VgprP1, Vgpr64}}})
1429 .Any(Init: {.Predicate: {UniP3}, .OperandMapping: {{SgprP3}, {SgprP3, Sgpr32}}})
1430 .Any(Init: {.Predicate: {DivP3}, .OperandMapping: {{VgprP3}, {VgprP3, Vgpr32}}});
1431
1432 addRulesForGOpcs(OpcList: {G_DYN_STACKALLOC})
1433 .Any(Init: {.Predicate: {UniP5, UniS32}, .OperandMapping: {{SgprP5}, {Sgpr32}, DynStackAlloc}})
1434 .Any(Init: {.Predicate: {UniP5, DivS32}, .OperandMapping: {{SgprP5}, {Vgpr32}, DynStackAlloc}});
1435
1436 addRulesForGOpcs(OpcList: {G_STACKRESTORE}).Any(Init: {.Predicate: {P5}, .OperandMapping: {{}, {SgprB32_ReadFirstLane}}});
1437
1438 addRulesForGOpcs(OpcList: {G_ABS}, FastTypes: Standard)
1439 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr32Trunc}, {Sgpr32SExt}})
1440 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16}, AbsToNegMax})
1441 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32}})
1442 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32}, AbsToNegMax})
1443 .Uni(Ty: V2S16, RuleApplyIDs: {{SgprV2S16}, {SgprV2S16}, AbsToS32})
1444 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16}, AbsToNegMax});
1445
1446 addRulesForGOpcs(OpcList: {G_BITREVERSE}, FastTypes: Standard)
1447 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32}})
1448 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32}})
1449 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {Sgpr64}})
1450 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64}});
1451
1452 addRulesForGOpcs(OpcList: {G_AMDGPU_FFBH_U32, G_AMDGPU_FFBL_B32, G_CTLZ_ZERO_POISON,
1453 G_CTTZ_ZERO_POISON})
1454 .Any(Init: {.Predicate: {UniS32, S32}, .OperandMapping: {{Sgpr32}, {Sgpr32}}})
1455 .Any(Init: {.Predicate: {DivS32, S32}, .OperandMapping: {{Vgpr32}, {Vgpr32}}})
1456 .Any(Init: {.Predicate: {UniS32, S64}, .OperandMapping: {{Sgpr32}, {Sgpr64}}})
1457 .Any(Init: {.Predicate: {DivS32, S64}, .OperandMapping: {{Vgpr32}, {Vgpr64}, SplitBitCount64To32}});
1458
1459 addRulesForGOpcs(OpcList: {G_CTPOP})
1460 .Any(Init: {.Predicate: {UniS32, S32}, .OperandMapping: {{Sgpr32}, {Sgpr32}}})
1461 .Any(Init: {.Predicate: {DivS32, S32}, .OperandMapping: {{Vgpr32}, {Vgpr32}}})
1462 .Any(Init: {.Predicate: {UniS32, S64}, .OperandMapping: {{Sgpr32}, {Sgpr64}}})
1463 .Any(Init: {.Predicate: {DivS32, S64}, .OperandMapping: {{Vgpr32}, {Vgpr64}, CtPop64To32}});
1464
1465 addRulesForGOpcs(OpcList: {G_FENCE}).Any(Init: {.Predicate: {{}}, .OperandMapping: {{}, {}}});
1466
1467 addRulesForGOpcs(OpcList: {G_READSTEADYCOUNTER, G_READCYCLECOUNTER}, FastTypes: Standard)
1468 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {}});
1469
1470 addRulesForGOpcs(OpcList: {G_GET_ROUNDING}, FastTypes: Standard)
1471 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {}, LowerGetRounding});
1472
1473 addRulesForGOpcs(OpcList: {G_SET_ROUNDING}, FastTypes: Standard)
1474 .Uni(Ty: S32, RuleApplyIDs: {{}, {SgprB32_ReadFirstLane}, LowerSetRounding})
1475 .Div(Ty: S32, RuleApplyIDs: {{}, {SgprB32_ReadFirstLane}, LowerSetRounding});
1476
1477 addRulesForGOpcs(OpcList: {G_WRITE_REGISTER})
1478 .Any(Init: {.Predicate: {_, B32}, .OperandMapping: {{}, {None, SgprB32_ReadFirstLane}}})
1479 .Any(Init: {.Predicate: {_, B64}, .OperandMapping: {{}, {None, SgprB64_ReadFirstLane}}});
1480
1481 addRulesForGOpcs(OpcList: {G_BLOCK_ADDR}).Any(Init: {.Predicate: {UniP0}, .OperandMapping: {{SgprP0}, {}}});
1482
1483 addRulesForGOpcs(OpcList: {G_GLOBAL_VALUE})
1484 .Any(Init: {.Predicate: {UniP0}, .OperandMapping: {{SgprP0}, {}}})
1485 .Any(Init: {.Predicate: {UniP1}, .OperandMapping: {{SgprP1}, {}}})
1486 .Any(Init: {.Predicate: {UniP3}, .OperandMapping: {{SgprP3}, {}}})
1487 .Any(Init: {.Predicate: {UniP4}, .OperandMapping: {{SgprP4}, {}}})
1488 .Any(Init: {.Predicate: {UniP8}, .OperandMapping: {{SgprP8}, {}}});
1489
1490 addRulesForGOpcs(OpcList: {G_AMDGPU_WAVE_ADDRESS}).Any(Init: {.Predicate: {UniP5}, .OperandMapping: {{SgprP5}, {}}});
1491
1492 addRulesForGOpcs(OpcList: {G_AMDGPU_SPONENTRY}, FastTypes: Standard).Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {}});
1493
1494 addRulesForGOpcs(OpcList: {G_SI_CALL})
1495 .Any(Init: {.Predicate: {_, UniP0}, .OperandMapping: {{None}, {SgprP0}}})
1496 .Any(Init: {.Predicate: {_, DivP0}, .OperandMapping: {{None}, {SgprP0Call_WF}}})
1497 .Any(Init: {.Predicate: {_, UniP4}, .OperandMapping: {{None}, {SgprP4}}})
1498 .Any(Init: {.Predicate: {_, DivP4}, .OperandMapping: {{None}, {SgprP4Call_WF}}});
1499
1500 bool hasSALUFloat = ST->hasSALUFloatInsts();
1501
1502 addRulesForGOpcs(OpcList: {G_FADD, G_FMUL, G_STRICT_FADD, G_STRICT_FMUL}, FastTypes: Standard)
1503 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16, Vgpr16}}, STPred: !hasSALUFloat)
1504 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {Sgpr16, Sgpr16}}, STPred: hasSALUFloat)
1505 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
1506 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}}, STPred: hasSALUFloat)
1507 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32}}, STPred: !hasSALUFloat)
1508 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
1509 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64, Vgpr64}})
1510 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr64}})
1511 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16}})
1512 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
1513 .Any(Init: {.Predicate: {UniV2S32}, .OperandMapping: {{UniInVgprV2S32}, {VgprV2S32, VgprV2S32}}})
1514 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {VgprV2S32, VgprV2S32}}})
1515 .Any(Init: {.Predicate: {UniV2S64}, .OperandMapping: {{UniInVgprV2S64}, {VgprV2S64, VgprV2S64}}})
1516 .Any(Init: {.Predicate: {DivV2S64}, .OperandMapping: {{VgprV2S64}, {VgprV2S64, VgprV2S64}}});
1517
1518 addRulesForGOpcs(OpcList: {G_FSUB, G_STRICT_FSUB}, FastTypes: Standard)
1519 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
1520 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
1521 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {Sgpr16, Sgpr16}}, STPred: hasSALUFloat)
1522 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16, Vgpr16}}, STPred: !hasSALUFloat)
1523 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}}, STPred: hasSALUFloat)
1524 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32}}, STPred: !hasSALUFloat)
1525 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16}})
1526 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}});
1527
1528 addRulesForGOpcs(OpcList: {G_FMAD}, FastTypes: Standard)
1529 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16, Vgpr16, Vgpr16}})
1530 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16, Vgpr16}})
1531 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32, Vgpr32}})
1532 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32, Vgpr32}});
1533
1534 addRulesForGOpcs(OpcList: {G_FLDEXP, G_STRICT_FLDEXP}, FastTypes: Standard)
1535 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32}})
1536 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
1537 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16, Vgpr16}})
1538 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
1539 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64, Vgpr32}})
1540 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr32}});
1541
1542 addRulesForGOpcs(OpcList: {G_FMA, G_STRICT_FMA}, FastTypes: Standard)
1543 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {Sgpr16, Sgpr16, Sgpr16}}, STPred: hasSALUFloat)
1544 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16, Vgpr16, Vgpr16}}, STPred: !hasSALUFloat)
1545 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16, Vgpr16}})
1546 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32, Sgpr32}}, STPred: hasSALUFloat)
1547 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32, Vgpr32}}, STPred: !hasSALUFloat)
1548 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32, Vgpr32}})
1549 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64, Vgpr64, Vgpr64}})
1550 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr64, Vgpr64}})
1551 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16, VgprV2S16}})
1552 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16, VgprV2S16}})
1553 .Any(Init: {.Predicate: {UniV2S32}, .OperandMapping: {{UniInVgprV2S32}, {VgprV2S32, VgprV2S32, VgprV2S32}}})
1554 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {VgprV2S32, VgprV2S32, VgprV2S32}}})
1555 .Any(Init: {.Predicate: {UniV2S64}, .OperandMapping: {{UniInVgprV2S64}, {VgprV2S64, VgprV2S64, VgprV2S64}}})
1556 .Any(Init: {.Predicate: {DivV2S64}, .OperandMapping: {{VgprV2S64}, {VgprV2S64, VgprV2S64, VgprV2S64}}});
1557
1558 addRulesForGOpcs(OpcList: {G_AMDGPU_FMED3}, FastTypes: Standard)
1559 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16, Vgpr16, Vgpr16}})
1560 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16, Vgpr16}})
1561 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32, Vgpr32}})
1562 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32, Vgpr32}});
1563
1564 // TODO: This opcode is generated from the i64->i16 signed clamped pattern in
1565 // the PreLegalizerCombiner. Move the combine to RegBankCombiner to keep more
1566 // instructions on SALU.
1567 addRulesForGOpcs(OpcList: {G_AMDGPU_SMED3}, FastTypes: Standard)
1568 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32, Vgpr32}})
1569 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32, Vgpr32}});
1570
1571 // FNEG and FABS are either folded as source modifiers or can be selected as
1572 // bitwise XOR and AND with Mask. XOR and AND are available on SALU but for
1573 // targets without SALU float we still select them as VGPR since there would
1574 // be no real sgpr use.
1575 addRulesForGOpcs(OpcList: {G_FNEG, G_FABS}, FastTypes: Standard)
1576 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16}}, STPred: !hasSALUFloat)
1577 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {Sgpr16}}, STPred: hasSALUFloat)
1578 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16}})
1579 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32}}, STPred: !hasSALUFloat)
1580 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32}}, STPred: hasSALUFloat)
1581 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32}})
1582 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64}})
1583 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64}})
1584 .Uni(Ty: V2S16, RuleApplyIDs: {{SgprV2S16}, {SgprV2S16}})
1585 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16}})
1586 .Any(Init: {.Predicate: {UniV2S32}, .OperandMapping: {{UniInVgprV2S32}, {VgprV2S32}}})
1587 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {VgprV2S32}}});
1588
1589 addRulesForGOpcs(OpcList: {G_FCANONICALIZE}, FastTypes: Standard)
1590 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32}})
1591 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32}})
1592 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16}})
1593 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16}})
1594 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64}})
1595 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64}})
1596 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {VgprV2S16}})
1597 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16}})
1598 .Any(Init: {.Predicate: {UniV2S32}, .OperandMapping: {{UniInVgprV2S32}, {VgprV2S32}}})
1599 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {VgprV2S32}}})
1600 .Any(Init: {.Predicate: {UniV2S64}, .OperandMapping: {{UniInVgprV2S64}, {VgprV2S64}}})
1601 .Any(Init: {.Predicate: {DivV2S64}, .OperandMapping: {{VgprV2S64}, {VgprV2S64}}});
1602
1603 bool hasPST = ST->hasPseudoScalarTrans();
1604 // Only 16-bit G_FSQRT operations are legal. F32/F64 is handled with a custom
1605 // legalization in the legalizer.
1606 addRulesForGOpcs(OpcList: {G_FSQRT})
1607 .Any(Init: {.Predicate: {DivS16}, .OperandMapping: {{Vgpr16}, {Vgpr16}}})
1608 .Any(Init: {.Predicate: {UniBF16}, .OperandMapping: {{UniInVgprS16}, {Vgpr16}}})
1609 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{Sgpr16}, {Sgpr16}}}, STPred: hasPST)
1610 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{UniInVgprS16}, {Vgpr16}}}, STPred: !hasPST);
1611
1612 addRulesForGOpcs(OpcList: {G_FPTOUI, G_FPTOSI, G_FPTOUI_SAT, G_FPTOSI_SAT})
1613 .Any(Init: {.Predicate: {UniS16, S16}, .OperandMapping: {{UniInVgprS16}, {Vgpr16}}})
1614 .Any(Init: {.Predicate: {DivS16, S16}, .OperandMapping: {{Vgpr16}, {Vgpr16}}})
1615 .Any(Init: {.Predicate: {UniS32, S16}, .OperandMapping: {{Sgpr32}, {Sgpr16}}}, STPred: hasSALUFloat)
1616 .Any(Init: {.Predicate: {UniS32, S16}, .OperandMapping: {{UniInVgprS32}, {Vgpr16}}}, STPred: !hasSALUFloat)
1617 .Any(Init: {.Predicate: {DivS32, S16}, .OperandMapping: {{Vgpr32}, {Vgpr16}}})
1618 .Any(Init: {.Predicate: {UniS16, S32}, .OperandMapping: {{Sgpr16}, {Sgpr32}}}, STPred: hasSALUFloat)
1619 .Any(Init: {.Predicate: {UniS16, S32}, .OperandMapping: {{UniInVgprS16}, {Vgpr32}}}, STPred: !hasSALUFloat)
1620 .Any(Init: {.Predicate: {DivS16, S32}, .OperandMapping: {{Vgpr16}, {Vgpr32}}})
1621 .Any(Init: {.Predicate: {UniS32, S32}, .OperandMapping: {{Sgpr32}, {Sgpr32}}}, STPred: hasSALUFloat)
1622 .Any(Init: {.Predicate: {UniS32, S32}, .OperandMapping: {{UniInVgprS32}, {Vgpr32}}}, STPred: !hasSALUFloat)
1623 .Any(Init: {.Predicate: {DivS32, S32}, .OperandMapping: {{Vgpr32}, {Vgpr32}}})
1624 .Any(Init: {.Predicate: {UniS32, S64}, .OperandMapping: {{UniInVgprS32}, {Vgpr64}}})
1625 .Any(Init: {.Predicate: {DivS32, S64}, .OperandMapping: {{Vgpr32}, {Vgpr64}}})
1626 .Any(Init: {.Predicate: {UniV2S16, V2S32}, .OperandMapping: {{UniInVgprV2S16}, {SgprV2S32}}})
1627 .Any(Init: {.Predicate: {DivV2S16, V2S32}, .OperandMapping: {{VgprV2S16}, {VgprV2S32}}});
1628
1629 addRulesForGOpcs(OpcList: {G_UITOFP, G_SITOFP})
1630 .Any(Init: {.Predicate: {UniS16, S16}, .OperandMapping: {{UniInVgprS16}, {Vgpr16}}})
1631 .Any(Init: {.Predicate: {DivS16, S16}, .OperandMapping: {{Vgpr16}, {Vgpr16}}})
1632 .Any(Init: {.Predicate: {UniS16, S32}, .OperandMapping: {{Sgpr16}, {Sgpr32}}}, STPred: hasSALUFloat)
1633 .Any(Init: {.Predicate: {UniS16, S32}, .OperandMapping: {{UniInVgprS16}, {Vgpr32}}}, STPred: !hasSALUFloat)
1634 .Any(Init: {.Predicate: {DivS16, S32}, .OperandMapping: {{Vgpr16}, {Vgpr32}}})
1635 .Any(Init: {.Predicate: {UniS32, S32}, .OperandMapping: {{Sgpr32}, {Sgpr32}}}, STPred: hasSALUFloat)
1636 .Any(Init: {.Predicate: {UniS32, S32}, .OperandMapping: {{UniInVgprS32}, {Vgpr32}}}, STPred: !hasSALUFloat)
1637 .Any(Init: {.Predicate: {DivS32, S32}, .OperandMapping: {{Vgpr32}, {Vgpr32}}})
1638 .Any(Init: {.Predicate: {UniS64, S32}, .OperandMapping: {{UniInVgprS64}, {Vgpr32}}})
1639 .Any(Init: {.Predicate: {DivS64, S32}, .OperandMapping: {{Vgpr64}, {Vgpr32}}});
1640
1641 addRulesForGOpcs(OpcList: {G_AMDGPU_S_BUFFER_PREFETCH})
1642 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {SgprV4S32_ReadFirstLane, Imm, SgprB32_ReadFirstLane}}});
1643
1644 Predicate IsDataPF([](const MachineInstr &MI) -> bool {
1645 // prefetch cache type: 0 == instruction (I$) prefetch, 1 == data prefetch.
1646 return MI.getOperand(i: 3).getImm() != 0;
1647 });
1648
1649 bool HasSMemPF = ST->hasSafeSmemPrefetch();
1650 bool HasVMemPF = ST->hasVmemPrefInsts();
1651 addRulesForGOpcs(OpcList: {G_PREFETCH})
1652 // Safe smem prefetch keeps both data and instruction prefetch.
1653 .Any(Init: {.Predicate: {UniPtr64}, .OperandMapping: {{}, {SgprPtr64}}}, STPred: HasSMemPF)
1654 // Vmem prefetch keeps data prefetch only.
1655 .Any(Init: {.Predicate: {{UniPtr64}, IsDataPF}, .OperandMapping: {{}, {SgprPtr64}}}, STPred: !HasSMemPF && HasVMemPF)
1656 .Any(Init: {.Predicate: {{UniPtr64}, IsDataPF}, .OperandMapping: {{}, {}, DeletePrefetch}},
1657 STPred: !HasSMemPF && !HasVMemPF)
1658 .Any(Init: {.Predicate: {{UniPtr64}, !IsDataPF}, .OperandMapping: {{}, {}, DeletePrefetch}}, STPred: !HasSMemPF)
1659
1660 .Any(Init: {.Predicate: {{DivPtr64}, IsDataPF}, .OperandMapping: {{}, {VgprPtr64}}}, STPred: HasVMemPF)
1661 // Otherwise prefetch the first active lane's address.
1662 .Any(Init: {.Predicate: {DivPtr64}, .OperandMapping: {{}, {SgprB64_ReadFirstLane}}}, STPred: HasSMemPF)
1663 .Any(Init: {.Predicate: {DivPtr64}, .OperandMapping: {{}, {}, DeletePrefetch}})
1664
1665 .Any(Init: {.Predicate: {Ptr32}, .OperandMapping: {{}, {}, DeletePrefetch}})
1666 .Any(Init: {.Predicate: {P8}, .OperandMapping: {{}, {}, DeletePrefetch}});
1667
1668 addRulesForGOpcs(OpcList: {G_FPEXT})
1669 .Any(Init: {.Predicate: {DivS32, S16}, .OperandMapping: {{Vgpr32}, {Vgpr16}}})
1670 .Any(Init: {.Predicate: {UniS64, S32}, .OperandMapping: {{UniInVgprS64}, {Vgpr32}}})
1671 .Any(Init: {.Predicate: {DivS64, S32}, .OperandMapping: {{Vgpr64}, {Vgpr32}}})
1672 .Any(Init: {.Predicate: {UniS32, S16}, .OperandMapping: {{Sgpr32}, {Sgpr16}}}, STPred: hasSALUFloat)
1673 .Any(Init: {.Predicate: {UniS32, S16}, .OperandMapping: {{UniInVgprS32}, {Vgpr16}}}, STPred: !hasSALUFloat);
1674
1675 addRulesForGOpcs(OpcList: {G_AMDGPU_CVT_PK_I16_I32}, FastTypes: Standard)
1676 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {Vgpr32, Vgpr32}})
1677 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {Vgpr32, Vgpr32}});
1678
1679 addRulesForGOpcs(OpcList: {G_AMDGPU_FMIN_LEGACY, G_AMDGPU_FMAX_LEGACY}, FastTypes: Standard)
1680 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32}})
1681 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}});
1682
1683 bool hasSALUMinimumMaximumInsts = ST->hasSALUMinimumMaximumInsts();
1684
1685 addRulesForGOpcs(OpcList: {G_FMINIMUM, G_FMAXIMUM}, FastTypes: Standard)
1686 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {Sgpr16, Sgpr16}}, STPred: hasSALUMinimumMaximumInsts)
1687 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16, Vgpr16}}, STPred: !hasSALUMinimumMaximumInsts)
1688 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
1689 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}}, STPred: hasSALUMinimumMaximumInsts)
1690 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32}}, STPred: !hasSALUMinimumMaximumInsts)
1691 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
1692 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64, Vgpr64}})
1693 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr64}})
1694 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16}})
1695 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}});
1696
1697 addRulesForGOpcs(OpcList: {G_FMINNUM_IEEE, G_FMAXNUM_IEEE}, FastTypes: Standard)
1698 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
1699 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
1700 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64, Vgpr64}})
1701 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr64}})
1702 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16}})
1703 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
1704 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {Sgpr16, Sgpr16}}, STPred: hasSALUFloat)
1705 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16, Vgpr16}}, STPred: !hasSALUFloat)
1706 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}}, STPred: hasSALUFloat)
1707 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32}}, STPred: !hasSALUFloat);
1708
1709 addRulesForGOpcs(OpcList: {G_FMINNUM, G_FMAXNUM, G_FMINIMUMNUM, G_FMAXIMUMNUM},
1710 FastTypes: Standard)
1711 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
1712 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
1713 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64, Vgpr64}})
1714 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr64}})
1715 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16}})
1716 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
1717 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {Sgpr16, Sgpr16}}, STPred: hasSALUFloat)
1718 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16, Vgpr16}}, STPred: !hasSALUFloat)
1719 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}}, STPred: hasSALUFloat)
1720 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32}}, STPred: !hasSALUFloat)
1721 .Any(Init: {.Predicate: {UniV2S64}, .OperandMapping: {{UniInVgprV2S64}, {VgprV2S64, VgprV2S64}}})
1722 .Any(Init: {.Predicate: {DivV2S64}, .OperandMapping: {{VgprV2S64}, {VgprV2S64, VgprV2S64}}});
1723
1724 addRulesForGOpcs(OpcList: {G_FPTRUNC})
1725 .Any(Init: {.Predicate: {DivS16, S32}, .OperandMapping: {{Vgpr16}, {Vgpr32}}})
1726 .Any(Init: {.Predicate: {UniS32, S64}, .OperandMapping: {{UniInVgprS32}, {Vgpr64}}})
1727 .Any(Init: {.Predicate: {DivS32, S64}, .OperandMapping: {{Vgpr32}, {Vgpr64}}})
1728 .Any(Init: {.Predicate: {UniV2S16, V2S32}, .OperandMapping: {{UniInVgprV2S16}, {VgprV2S32}}})
1729 .Any(Init: {.Predicate: {DivV2S16, V2S32}, .OperandMapping: {{VgprV2S16}, {VgprV2S32}}})
1730 .Any(Init: {.Predicate: {UniS16, S32}, .OperandMapping: {{Sgpr16}, {Sgpr32}}}, STPred: hasSALUFloat)
1731 .Any(Init: {.Predicate: {UniS16, S32}, .OperandMapping: {{UniInVgprS16}, {Vgpr32}}}, STPred: !hasSALUFloat);
1732
1733 addRulesForGOpcs(OpcList: {G_INTRINSIC_FPTRUNC_ROUND})
1734 .Any(Init: {.Predicate: {UniS16, S32}, .OperandMapping: {{Sgpr16}, {Sgpr32}}}, STPred: hasSALUFloat)
1735 .Any(Init: {.Predicate: {UniS16, S32}, .OperandMapping: {{UniInVgprS16}, {Vgpr32}}}, STPred: !hasSALUFloat)
1736 .Any(Init: {.Predicate: {DivS16, S32}, .OperandMapping: {{Vgpr16}, {Vgpr32}}})
1737 .Any(Init: {.Predicate: {UniS16, S64}, .OperandMapping: {{UniInVgprS16}, {Vgpr64}}})
1738 .Any(Init: {.Predicate: {DivS16, S64}, .OperandMapping: {{Vgpr16}, {Vgpr64}}})
1739 .Any(Init: {.Predicate: {UniS32, S64}, .OperandMapping: {{UniInVgprS32}, {Vgpr64}}})
1740 .Any(Init: {.Predicate: {DivS32, S64}, .OperandMapping: {{Vgpr32}, {Vgpr64}}});
1741
1742 addRulesForGOpcs(OpcList: {G_IS_FPCLASS})
1743 .Any(Init: {.Predicate: {DivS1, S16}, .OperandMapping: {{Vcc}, {Vgpr16}}})
1744 .Any(Init: {.Predicate: {UniS1, S16}, .OperandMapping: {{UniInVcc}, {Vgpr16}}})
1745 .Any(Init: {.Predicate: {DivS1, S32}, .OperandMapping: {{Vcc}, {Vgpr32}}})
1746 .Any(Init: {.Predicate: {UniS1, S32}, .OperandMapping: {{UniInVcc}, {Vgpr32}}})
1747 .Any(Init: {.Predicate: {DivS1, S64}, .OperandMapping: {{Vcc}, {Vgpr64}}})
1748 .Any(Init: {.Predicate: {UniS1, S64}, .OperandMapping: {{UniInVcc}, {Vgpr64}}});
1749
1750 addRulesForGOpcs(OpcList: {G_FCMP}, FastTypes: Standard)
1751 .Any(Init: {.Predicate: {UniS1, _, S16}, .OperandMapping: {{Sgpr32Trunc}, {None, Sgpr16, Sgpr16}}},
1752 STPred: hasSALUFloat)
1753 .Any(Init: {.Predicate: {UniS1, _, S16}, .OperandMapping: {{UniInVcc}, {None, Vgpr16, Vgpr16}}},
1754 STPred: !hasSALUFloat)
1755 .Any(Init: {.Predicate: {DivS1, _, S16}, .OperandMapping: {{Vcc}, {None, Vgpr16, Vgpr16}}})
1756 .Any(Init: {.Predicate: {UniS1, _, S32}, .OperandMapping: {{Sgpr32Trunc}, {None, Sgpr32, Sgpr32}}},
1757 STPred: hasSALUFloat)
1758 .Any(Init: {.Predicate: {UniS1, _, S32}, .OperandMapping: {{UniInVcc}, {None, Vgpr32, Vgpr32}}},
1759 STPred: !hasSALUFloat)
1760 .Any(Init: {.Predicate: {DivS1, _, S32}, .OperandMapping: {{Vcc}, {None, Vgpr32, Vgpr32}}})
1761 .Any(Init: {.Predicate: {UniS1, _, S64}, .OperandMapping: {{UniInVcc}, {None, Vgpr64, Vgpr64}}})
1762 .Any(Init: {.Predicate: {DivS1, _, S64}, .OperandMapping: {{Vcc}, {None, Vgpr64, Vgpr64}}});
1763
1764 addRulesForGOpcs(OpcList: {G_INTRINSIC_ROUNDEVEN, G_FEXP2, G_FLOG2}, FastTypes: Standard)
1765 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16}})
1766 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16}})
1767 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32}})
1768 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32}})
1769 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64}})
1770 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64}});
1771
1772 addRulesForGOpcs(OpcList: {G_INTRINSIC_TRUNC, G_FFLOOR, G_FCEIL}, FastTypes: Standard)
1773 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16}})
1774 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {Sgpr16}}, STPred: hasSALUFloat)
1775 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16}})
1776 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32}})
1777 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32}}, STPred: hasSALUFloat)
1778 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32}})
1779 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64}})
1780 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64}});
1781
1782 addRulesForGOpcs(OpcList: {G_AMDGPU_GLOBAL_LOAD_MONITOR, G_AMDGPU_FLAT_LOAD_MONITOR},
1783 FastTypes: StandardB)
1784 .Uni(Ty: B32, RuleApplyIDs: {{UniInVgprB32}, {SgprPtr64}})
1785 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {VgprPtr64}})
1786 .Uni(Ty: B64, RuleApplyIDs: {{UniInVgprB64}, {SgprPtr64}})
1787 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {VgprPtr64}})
1788 .Uni(Ty: B128, RuleApplyIDs: {{UniInVgprB128}, {SgprPtr64}})
1789 .Div(Ty: B128, RuleApplyIDs: {{VgprB128}, {VgprPtr64}});
1790
1791 addRulesForGOpcs(OpcList: {G_AMDGPU_WHOLE_WAVE_FUNC_SETUP})
1792 .Any(Init: {.Predicate: {DivS1}, .OperandMapping: {{Vcc}, {}}});
1793
1794 addRulesForGOpcs(OpcList: {G_AMDGPU_WHOLE_WAVE_FUNC_RETURN}).Any(Init: {.Predicate: {}, .OperandMapping: {{}, {Vcc}}});
1795
1796 using namespace Intrinsic;
1797
1798 addRulesForIOpcs(OpcList: {returnaddress}).Any(Init: {.Predicate: {UniP0}, .OperandMapping: {{SgprP0}, {}}});
1799
1800 addRulesForIOpcs(OpcList: {amdgcn_s_bitreplicate}, FastTypes: Standard)
1801 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {IntrId, Sgpr32}})
1802 .Div(Ty: S64, RuleApplyIDs: {{Sgpr64ToVgprDst}, {IntrId, SgprB32_ReadFirstLane}});
1803
1804 addRulesForIOpcs(OpcList: {amdgcn_s_getpc}).Any(Init: {.Predicate: {UniS64, _}, .OperandMapping: {{Sgpr64}, {}}});
1805
1806 addRulesForIOpcs(OpcList: {amdgcn_s_getreg}).Any(Init: {.Predicate: {}, .OperandMapping: {{Sgpr32}, {IntrId}}});
1807
1808 addRulesForIOpcs(OpcList: {amdgcn_s_setreg})
1809 .Any(Init: {.Predicate: {_, _, S32}, .OperandMapping: {{}, {IntrId, Imm, SgprB32_ReadFirstLane}}});
1810
1811 addRulesForIOpcs(OpcList: {amdgcn_s_sendmsg, amdgcn_s_sendmsghalt})
1812 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, Imm, SgprB32_M0}}});
1813
1814 addRulesForIOpcs(OpcList: {amdgcn_s_sendmsg_rtn})
1815 .Any(Init: {.Predicate: {S32}, .OperandMapping: {{Sgpr32}, {}}})
1816 .Any(Init: {.Predicate: {S64}, .OperandMapping: {{Sgpr64}, {}}});
1817
1818 addRulesForIOpcs(OpcList: {amdgcn_s_memrealtime, amdgcn_s_memtime}, FastTypes: Standard)
1819 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {IntrId}});
1820
1821 addRulesForIOpcs(OpcList: {amdgcn_groupstaticsize, amdgcn_pops_exiting_wave_id,
1822 amdgcn_reloc_constant, amdgcn_s_get_waveid_in_workgroup},
1823 FastTypes: Standard)
1824 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {IntrId}});
1825
1826 // Intrinsics with no register operands.
1827 addRulesForIOpcs(OpcList: {amdgcn_asyncmark,
1828 amdgcn_endpgm,
1829 amdgcn_iglp_opt,
1830 amdgcn_init_exec,
1831 amdgcn_s_barrier,
1832 amdgcn_s_barrier_leave,
1833 amdgcn_s_barrier_signal,
1834 amdgcn_s_barrier_wait,
1835 amdgcn_s_decperflevel,
1836 amdgcn_s_incperflevel,
1837 amdgcn_s_monitor_sleep,
1838 amdgcn_s_nop,
1839 amdgcn_s_sethalt,
1840 amdgcn_s_setprio,
1841 amdgcn_s_setprio_inc_wg,
1842 amdgcn_s_sleep,
1843 amdgcn_s_ttracedata_imm,
1844 amdgcn_s_wait_asynccnt,
1845 amdgcn_s_wait_bvhcnt,
1846 amdgcn_s_wait_dscnt,
1847 amdgcn_s_wait_event,
1848 amdgcn_s_wait_event_export_ready,
1849 amdgcn_s_wait_expcnt,
1850 amdgcn_s_wait_kmcnt,
1851 amdgcn_s_wait_loadcnt,
1852 amdgcn_s_wait_samplecnt,
1853 amdgcn_s_wait_storecnt,
1854 amdgcn_s_wait_tensorcnt,
1855 amdgcn_s_waitcnt,
1856 amdgcn_sched_barrier,
1857 amdgcn_sched_group_barrier,
1858 amdgcn_unreachable,
1859 amdgcn_wait_asyncmark,
1860 amdgcn_wave_barrier})
1861 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {}}});
1862
1863 addRulesForIOpcs(OpcList: {amdgcn_init_exec_from_input})
1864 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, Sgpr32}}});
1865
1866 addRulesForIOpcs(OpcList: {amdgcn_s_ttracedata}).Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, SgprB32_M0}}});
1867
1868 addRulesForIOpcs(OpcList: {amdgcn_s_sleep_var})
1869 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, SgprB32_ReadFirstLane}}});
1870
1871 addRulesForIOpcs(OpcList: {amdgcn_s_barrier_join, amdgcn_s_wakeup_barrier})
1872 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, SgprB32_M0}}});
1873
1874 addRulesForIOpcs(OpcList: {amdgcn_s_barrier_signal_var, amdgcn_s_barrier_init})
1875 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, SgprB32_M0, SgprB32_M0}}});
1876
1877 addRulesForIOpcs(OpcList: {amdgcn_s_barrier_signal_isfirst})
1878 .Any(Init: {.Predicate: {UniS1}, .OperandMapping: {{Sgpr32Trunc}, {}}});
1879
1880 addRulesForIOpcs(
1881 OpcList: {amdgcn_s_get_named_barrier_state, amdgcn_s_get_barrier_state}, FastTypes: Standard)
1882 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {IntrId, SgprB32_M0}});
1883
1884 addRulesForIOpcs(OpcList: {amdgcn_flat_prefetch}).Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, VgprP0}}});
1885
1886 addRulesForIOpcs(OpcList: {amdgcn_global_prefetch}).Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, VgprP1}}});
1887
1888 addRulesForIOpcs(OpcList: {amdgcn_s_prefetch_data, amdgcn_s_prefetch_inst})
1889 .Any(Init: {.Predicate: {_, Ptr32}, .OperandMapping: {{}, {}, DeletePrefetch}})
1890 .Any(Init: {.Predicate: {_, P8}, .OperandMapping: {{}, {}, DeletePrefetch}})
1891 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, SgprB64_ReadFirstLane, SgprB32_ReadFirstLane}}});
1892
1893 addRulesForIOpcs(OpcList: {amdgcn_class})
1894 .Any(Init: {.Predicate: {UniS1, _, S16}, .OperandMapping: {{UniInVcc}, {IntrId, Vgpr16, Vgpr32}}})
1895 .Any(Init: {.Predicate: {DivS1, _, S16}, .OperandMapping: {{Vcc}, {IntrId, Vgpr16, Vgpr32}}})
1896 .Any(Init: {.Predicate: {UniS1, _, S32}, .OperandMapping: {{UniInVcc}, {IntrId, Vgpr32, Vgpr32}}})
1897 .Any(Init: {.Predicate: {DivS1, _, S32}, .OperandMapping: {{Vcc}, {IntrId, Vgpr32, Vgpr32}}})
1898 .Any(Init: {.Predicate: {UniS1, _, S64}, .OperandMapping: {{UniInVcc}, {IntrId, Vgpr64, Vgpr32}}})
1899 .Any(Init: {.Predicate: {DivS1, _, S64}, .OperandMapping: {{Vcc}, {IntrId, Vgpr64, Vgpr32}}});
1900
1901 // This is "intrinsic lane mask" it was set to i32/i64 in llvm-ir.
1902 addRulesForIOpcs(OpcList: {amdgcn_end_cf})
1903 .Any(Init: {.Predicate: {_, UniS32}, .OperandMapping: {{}, {IntrId, Sgpr32}}})
1904 .Any(Init: {.Predicate: {_, UniS64}, .OperandMapping: {{}, {IntrId, Sgpr64}}});
1905
1906 addRulesForIOpcs(OpcList: {amdgcn_if_break}, FastTypes: Standard)
1907 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {IntrId, Vcc, Sgpr64}})
1908 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {IntrId, Vcc, Sgpr32}});
1909
1910 addRulesForIOpcs(OpcList: {amdgcn_exp})
1911 .Any(Init: {.Predicate: {_, _, _, S32, S32, S32, S32},
1912 .OperandMapping: {{}, {IntrId, Imm, Imm, Vgpr32, Vgpr32, Vgpr32, Vgpr32}}});
1913
1914 addRulesForIOpcs(OpcList: {amdgcn_exp_compr})
1915 .Any(Init: {.Predicate: {_, _, _, V2S16}, .OperandMapping: {{}, {IntrId, Imm, Imm, VgprV2S16, VgprV2S16}}});
1916
1917 addRulesForIOpcs(OpcList: {amdgcn_exp_row})
1918 .Any(Init: {.Predicate: {_, _, _, S32, S32, S32, S32, _, S32},
1919 .OperandMapping: {{},
1920 {IntrId, Imm, Imm, Vgpr32, Vgpr32, Vgpr32, Vgpr32, Imm,
1921 SgprB32_M0}}});
1922
1923 addRulesForIOpcs(OpcList: {amdgcn_lds_direct_load}, FastTypes: StandardB)
1924 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {IntrId, SgprB32_M0}});
1925
1926 addRulesForIOpcs(OpcList: {amdgcn_lds_param_load}, FastTypes: Standard)
1927 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Imm, Imm, SgprB32_M0}});
1928
1929 addRulesForIOpcs(OpcList: {amdgcn_mbcnt_lo, amdgcn_mbcnt_hi}, FastTypes: Standard)
1930 .Div(Ty: S32, RuleApplyIDs: {{}, {Vgpr32, None, Vgpr32, Vgpr32}});
1931
1932 addRulesForIOpcs(OpcList: {amdgcn_readfirstlane})
1933 .Any(Init: {.Predicate: {UniB32, _, DivB32}, .OperandMapping: {{}, {SgprB32, None, VgprB32}}})
1934 // this should not exist in the first place, it is from call lowering
1935 // readfirstlaning just in case register is not in sgpr.
1936 .Any(Init: {.Predicate: {UniS32, _, UniS32}, .OperandMapping: {{}, {Sgpr32, None, Vgpr32}}});
1937
1938 addRulesForIOpcs(OpcList: {amdgcn_readlane}, FastTypes: StandardB)
1939 .Uni(Ty: B32, RuleApplyIDs: {{SgprB32}, {IntrId, VgprB32, SgprB32_ReadFirstLane}});
1940
1941 addRulesForIOpcs(OpcList: {amdgcn_s_quadmask, amdgcn_s_wqm}, FastTypes: StandardB)
1942 .Uni(Ty: B32, RuleApplyIDs: {{SgprB32}, {IntrId, SgprB32_ReadFirstLane}})
1943 .Uni(Ty: B64, RuleApplyIDs: {{SgprB64}, {IntrId, SgprB64_ReadFirstLane}});
1944
1945 addRulesForIOpcs(OpcList: {amdgcn_writelane}, FastTypes: StandardB)
1946 .Div(Ty: B32,
1947 RuleApplyIDs: {{VgprB32},
1948 {IntrId, SgprB32_ReadFirstLane, SgprB32_ReadFirstLane, VgprB32}});
1949
1950 addRulesForIOpcs(OpcList: {amdgcn_add_max_i32, amdgcn_add_max_u32, amdgcn_add_min_i32,
1951 amdgcn_add_min_u32},
1952 FastTypes: Standard)
1953 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
1954 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
1955
1956 addRulesForIOpcs(OpcList: {amdgcn_pk_add_max_i16, amdgcn_pk_add_max_u16,
1957 amdgcn_pk_add_min_i16, amdgcn_pk_add_min_u16},
1958 FastTypes: Standard)
1959 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {IntrId, VgprV2S16, VgprV2S16, VgprV2S16}})
1960 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {IntrId, VgprV2S16, VgprV2S16, VgprV2S16}});
1961
1962 addRulesForIOpcs(OpcList: {amdgcn_permlane16, amdgcn_permlanex16}, FastTypes: StandardB)
1963 .Div(Ty: B32, RuleApplyIDs: {{VgprB32},
1964 {IntrId, VgprB32, VgprB32, SgprB32_ReadFirstLane,
1965 SgprB32_ReadFirstLane}});
1966
1967 addRulesForIOpcs(OpcList: {amdgcn_permlane_bcast, amdgcn_permlane_up,
1968 amdgcn_permlane_down, amdgcn_permlane_xor},
1969 FastTypes: StandardB)
1970 .Div(Ty: B32,
1971 RuleApplyIDs: {{VgprB32},
1972 {IntrId, VgprB32, SgprB32_ReadFirstLane, SgprB32_ReadFirstLane}});
1973
1974 addRulesForIOpcs(OpcList: {amdgcn_permlane_idx_gen}, FastTypes: Standard)
1975 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, SgprB32_ReadFirstLane}});
1976
1977 addRulesForIOpcs(OpcList: {amdgcn_alignbyte, amdgcn_perm}, FastTypes: Standard)
1978 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
1979 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
1980
1981 addRulesForIOpcs(OpcList: {amdgcn_mqsad_pk_u16_u8, amdgcn_qsad_pk_u16_u8}, FastTypes: Standard)
1982 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {IntrId, Vgpr64, Vgpr32, Vgpr64}})
1983 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {IntrId, Vgpr64, Vgpr32, Vgpr64}});
1984
1985 addRulesForIOpcs(OpcList: {amdgcn_mqsad_u32_u8})
1986 .Any(
1987 Init: {.Predicate: {UniV4S32}, .OperandMapping: {{UniInVgprV4S32}, {IntrId, Vgpr64, Vgpr32, VgprV4S32}}})
1988 .Any(Init: {.Predicate: {DivV4S32}, .OperandMapping: {{VgprV4S32}, {IntrId, Vgpr64, Vgpr32, VgprV4S32}}});
1989
1990 addRulesForIOpcs(OpcList: {amdgcn_lerp}, FastTypes: Standard)
1991 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
1992 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
1993
1994 addRulesForIOpcs(
1995 OpcList: {amdgcn_msad_u8, amdgcn_sad_hi_u8, amdgcn_sad_u16, amdgcn_sad_u8},
1996 FastTypes: Standard)
1997 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
1998 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
1999
2000 addRulesForIOpcs(
2001 OpcList: {amdgcn_wave_reduce_add, amdgcn_wave_reduce_and, amdgcn_wave_reduce_fadd,
2002 amdgcn_wave_reduce_fmax, amdgcn_wave_reduce_fmin,
2003 amdgcn_wave_reduce_fsub, amdgcn_wave_reduce_max, amdgcn_wave_reduce_min,
2004 amdgcn_wave_reduce_or, amdgcn_wave_reduce_sub, amdgcn_wave_reduce_umax,
2005 amdgcn_wave_reduce_umin, amdgcn_wave_reduce_xor},
2006 FastTypes: Standard)
2007 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {IntrId, Sgpr32}})
2008 .Div(Ty: S32, RuleApplyIDs: {{Sgpr32ToVgprDst}, {IntrId, VgprB32}})
2009 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {IntrId, Sgpr64}})
2010 .Div(Ty: S64, RuleApplyIDs: {{Sgpr64ToVgprDst}, {IntrId, VgprB64}});
2011
2012 addRulesForIOpcs(OpcList: {amdgcn_wave_shuffle}, FastTypes: Standard)
2013 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32}})
2014 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32}});
2015
2016 addRulesForIOpcs(OpcList: {amdgcn_bitop3, amdgcn_fmad_ftz}, FastTypes: Standard)
2017 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, Vgpr16, Vgpr16, Vgpr16}})
2018 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, Vgpr16, Vgpr16, Vgpr16}})
2019 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
2020 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
2021
2022 addRulesForIOpcs(OpcList: {amdgcn_udot4, amdgcn_sdot4, amdgcn_udot8, amdgcn_sdot8,
2023 amdgcn_dot4_f32_bf8_bf8, amdgcn_dot4_f32_bf8_fp8,
2024 amdgcn_dot4_f32_fp8_fp8, amdgcn_dot4_f32_fp8_bf8},
2025 FastTypes: Standard)
2026 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
2027 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
2028
2029 addRulesForIOpcs(OpcList: {amdgcn_rsq, amdgcn_rsq_clamp})
2030 .Any(Init: {.Predicate: {UniBF16}, .OperandMapping: {{UniInVgprS16}, {IntrId, Vgpr16}}})
2031 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{Sgpr16}, {IntrId, Sgpr16}}}, STPred: hasPST)
2032 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{UniInVgprS16}, {IntrId, Vgpr16}}}, STPred: !hasPST)
2033 .Any(Init: {.Predicate: {DivS16}, .OperandMapping: {{Vgpr16}, {IntrId, Vgpr16}}})
2034 .Any(Init: {.Predicate: {UniS32}, .OperandMapping: {{Sgpr32}, {IntrId, Sgpr32}}}, STPred: hasPST)
2035 .Any(Init: {.Predicate: {UniS32}, .OperandMapping: {{UniInVgprS32}, {IntrId, Vgpr32}}}, STPred: !hasPST)
2036 .Any(Init: {.Predicate: {DivS32}, .OperandMapping: {{Vgpr32}, {IntrId, Vgpr32}}})
2037 .Any(Init: {.Predicate: {UniS64}, .OperandMapping: {{UniInVgprS64}, {IntrId, Vgpr64}}})
2038 .Any(Init: {.Predicate: {DivS64}, .OperandMapping: {{Vgpr64}, {IntrId, Vgpr64}}});
2039
2040 addRulesForIOpcs(OpcList: {amdgcn_mul_u24, amdgcn_mul_i24}, FastTypes: Standard)
2041 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32}})
2042 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32}})
2043 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {IntrId, Vgpr32, Vgpr32}})
2044 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {IntrId, Vgpr32, Vgpr32}});
2045
2046 addRulesForIOpcs(OpcList: {amdgcn_ds_bpermute, amdgcn_ds_bpermute_fi_b32,
2047 amdgcn_ds_permute, amdgcn_fmul_legacy, amdgcn_mulhi_i24,
2048 amdgcn_mulhi_u24},
2049 FastTypes: Standard)
2050 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32}})
2051 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32}});
2052
2053 addRulesForIOpcs(OpcList: {amdgcn_cvt_sr_bf8_f32, amdgcn_cvt_sr_fp8_f32,
2054 amdgcn_cvt_sr_fp8_f32_e5m3, amdgcn_cvt_pk_bf8_f32,
2055 amdgcn_cvt_pk_fp8_f32, amdgcn_cvt_pk_fp8_f32_e5m3,
2056 amdgcn_cvt_pk_u8_f32},
2057 FastTypes: Standard)
2058 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
2059 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
2060
2061 addRulesForIOpcs(OpcList: {amdgcn_cvt_off_f32_i4, amdgcn_cvt_f32_bf8,
2062 amdgcn_cvt_f32_fp8, amdgcn_cvt_f32_fp8_e5m3},
2063 FastTypes: Standard)
2064 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32}})
2065 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32}});
2066
2067 addRulesForIOpcs(OpcList: {amdgcn_cvt_pk_f32_bf8, amdgcn_cvt_pk_f32_fp8})
2068 .Any(Init: {.Predicate: {UniV2S32}, .OperandMapping: {{UniInVgprV2S32}, {IntrId, Vgpr32}}})
2069 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {IntrId, Vgpr32}}});
2070
2071 addRulesForIOpcs(OpcList: {amdgcn_cvt_f16_bf8, amdgcn_cvt_f16_fp8}, FastTypes: Standard)
2072 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, Vgpr32}})
2073 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, Vgpr32}});
2074
2075 addRulesForIOpcs(OpcList: {amdgcn_cvt_pk_f16_bf8, amdgcn_cvt_pk_f16_fp8}, FastTypes: Standard)
2076 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {IntrId, Vgpr16}})
2077 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {IntrId, Vgpr16}});
2078
2079 addRulesForIOpcs(OpcList: {amdgcn_cvt_pk_bf8_f16, amdgcn_cvt_pk_fp8_f16}, FastTypes: Standard)
2080 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, VgprV2S16}})
2081 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, VgprV2S16}});
2082
2083 addRulesForIOpcs(OpcList: {amdgcn_cvt_sr_bf8_f16, amdgcn_cvt_sr_fp8_f16}, FastTypes: Standard)
2084 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr16, Vgpr32, Vgpr32}})
2085 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr16, Vgpr32, Vgpr32}});
2086
2087 addRulesForIOpcs(OpcList: {amdgcn_cvt_sr_pk_f16_f32, amdgcn_cvt_sr_pk_bf16_f32},
2088 FastTypes: Standard)
2089 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
2090 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
2091
2092 addRulesForIOpcs(
2093 OpcList: {amdgcn_cvt_scalef32_sr_fp8_f16, amdgcn_cvt_scalef32_sr_bf8_f16,
2094 amdgcn_cvt_scalef32_sr_fp8_bf16, amdgcn_cvt_scalef32_sr_bf8_bf16})
2095 .Any(Init: {.Predicate: {DivS32}, .OperandMapping: {{Vgpr32}, {IntrId, Vgpr32, Vgpr16, Vgpr32, Vgpr32}}})
2096 .Any(Init: {.Predicate: {UniS32},
2097 .OperandMapping: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr16, Vgpr32, Vgpr32}}});
2098
2099 addRulesForIOpcs(
2100 OpcList: {amdgcn_cvt_scalef32_sr_fp8_f32, amdgcn_cvt_scalef32_sr_bf8_f32})
2101 .Any(Init: {.Predicate: {DivS32}, .OperandMapping: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32, Vgpr32}}})
2102 .Any(Init: {.Predicate: {UniS32},
2103 .OperandMapping: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32, Vgpr32}}});
2104
2105 addRulesForIOpcs(OpcList: {amdgcn_cubesc, amdgcn_cubetc, amdgcn_cubema, amdgcn_cubeid,
2106 amdgcn_fma_legacy},
2107 FastTypes: Standard)
2108 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
2109 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
2110
2111 addRulesForIOpcs(OpcList: {amdgcn_frexp_mant, amdgcn_fract}, FastTypes: Standard)
2112 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, Vgpr16}})
2113 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, Vgpr16}})
2114 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32}})
2115 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32}})
2116 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {IntrId, Vgpr64}})
2117 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {IntrId, Vgpr64}});
2118
2119 addRulesForIOpcs(OpcList: {amdgcn_prng_b32})
2120 .Any(Init: {.Predicate: {UniS32}, .OperandMapping: {{UniInVgprS32}, {IntrId, Vgpr32}}})
2121 .Any(Init: {.Predicate: {DivS32}, .OperandMapping: {{Vgpr32}, {IntrId, Vgpr32}}});
2122
2123 addRulesForIOpcs(
2124 OpcList: {amdgcn_exclusive_scan_sum_i32, amdgcn_exclusive_scan_sum_u32}, FastTypes: Standard)
2125 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Imm}})
2126 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Imm}});
2127
2128 addRulesForIOpcs(
2129 OpcList: {amdgcn_exclusive_scan_xor_b32, amdgcn_exclusive_scan_or_b32,
2130 amdgcn_exclusive_scan_and_b32, amdgcn_exclusive_scan_min_i32,
2131 amdgcn_exclusive_scan_min_u32, amdgcn_exclusive_scan_max_i32,
2132 amdgcn_exclusive_scan_max_u32},
2133 FastTypes: Standard)
2134 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32}})
2135 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32}});
2136
2137 addRulesForIOpcs(
2138 OpcList: {amdgcn_exclusive_scan_min_i16, amdgcn_exclusive_scan_min_u16,
2139 amdgcn_exclusive_scan_max_i16, amdgcn_exclusive_scan_max_u16},
2140 FastTypes: Standard)
2141 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, Vgpr16, Vgpr32}})
2142 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, Vgpr16, Vgpr32}});
2143
2144 addRulesForIOpcs(OpcList: {amdgcn_wave_match_b32})
2145 .Any(Init: {.Predicate: {UniS32}, .OperandMapping: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32}}})
2146 .Any(Init: {.Predicate: {DivS32}, .OperandMapping: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32}}});
2147
2148 addRulesForIOpcs(OpcList: {amdgcn_sffbh}, FastTypes: Standard)
2149 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {IntrId, Sgpr32}})
2150 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32}});
2151
2152 addRulesForIOpcs(OpcList: {amdgcn_ubfe, amdgcn_sbfe}, FastTypes: Standard)
2153 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
2154 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {IntrId, Sgpr32, Sgpr32, Sgpr32}, S_BFE})
2155 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {IntrId, Sgpr64, Sgpr32, Sgpr32}, S_BFE})
2156 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {IntrId, Vgpr64, Vgpr32, Vgpr32}, V_BFE});
2157
2158 addRulesForIOpcs(OpcList: {amdgcn_cvt_pk_i16, amdgcn_cvt_pk_u16, amdgcn_cvt_pknorm_i16,
2159 amdgcn_cvt_pknorm_u16},
2160 FastTypes: Standard)
2161 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {IntrId, Vgpr32, Vgpr32}})
2162 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {IntrId, Vgpr32, Vgpr32}});
2163
2164 addRulesForIOpcs(OpcList: {amdgcn_cvt_pkrtz}, FastTypes: Standard)
2165 .Uni(Ty: V2S16, RuleApplyIDs: {{SgprV2S16}, {IntrId, Sgpr32, Sgpr32}}, STPred: hasSALUFloat)
2166 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {IntrId, Vgpr32, Vgpr32}}, STPred: !hasSALUFloat)
2167 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {IntrId, Vgpr32, Vgpr32}});
2168
2169 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_sr_pk32_bf6_f16,
2170 amdgcn_cvt_scalef32_sr_pk32_fp6_f16,
2171 amdgcn_cvt_scalef32_sr_pk32_bf6_bf16,
2172 amdgcn_cvt_scalef32_sr_pk32_fp6_bf16},
2173 FastTypes: Standard)
2174 .Any(Init: {.Predicate: {DivV6S32}, .OperandMapping: {{VgprV6S32}, {IntrId, VgprV32S16, Vgpr32, Vgpr32}}})
2175 .Any(Init: {.Predicate: {UniV6S32},
2176 .OperandMapping: {{UniInVgprV6S32}, {IntrId, VgprV32S16, Vgpr32, Vgpr32}}});
2177
2178 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_sr_pk32_bf6_f32,
2179 amdgcn_cvt_scalef32_sr_pk32_fp6_f32},
2180 FastTypes: Standard)
2181 .Any(Init: {.Predicate: {DivV6S32}, .OperandMapping: {{VgprV6S32}, {IntrId, VgprV32S32, Vgpr32, Vgpr32}}})
2182 .Any(Init: {.Predicate: {UniV6S32},
2183 .OperandMapping: {{UniInVgprV6S32}, {IntrId, VgprV32S32, Vgpr32, Vgpr32}}});
2184
2185 addRulesForIOpcs(
2186 OpcList: {amdgcn_cvt_scalef32_sr_pk_fp4_f16, amdgcn_cvt_scalef32_sr_pk_fp4_bf16},
2187 FastTypes: Standard)
2188 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, VgprV2S16, Vgpr32, Vgpr32}})
2189 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, VgprV2S16, Vgpr32, Vgpr32}});
2190
2191 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_sr_pk_fp4_f32}, FastTypes: Standard)
2192 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, VgprV2S32, Vgpr32, Vgpr32}})
2193 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, VgprV2S32, Vgpr32, Vgpr32}});
2194
2195 addRulesForIOpcs(
2196 OpcList: {amdgcn_cvt_scalef32_2xpk16_fp6_f32, amdgcn_cvt_scalef32_2xpk16_bf6_f32})
2197 .Any(
2198 Init: {.Predicate: {DivV6S32}, .OperandMapping: {{VgprV6S32}, {IntrId, VgprV16S32, VgprV16S32, Vgpr32}}})
2199 .Any(Init: {.Predicate: {UniV6S32},
2200 .OperandMapping: {{UniInVgprV6S32}, {IntrId, VgprV16S32, VgprV16S32, Vgpr32}}});
2201
2202 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_f16_fp8, amdgcn_cvt_scalef32_f16_bf8},
2203 FastTypes: Standard)
2204 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {IntrId, VgprV2S16, Vgpr32, Vgpr32}})
2205 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {IntrId, VgprV2S16, Vgpr32, Vgpr32}});
2206
2207 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_f32_fp8, amdgcn_cvt_scalef32_f32_bf8},
2208 FastTypes: Standard)
2209 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32}})
2210 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32}});
2211
2212 addRulesForIOpcs(
2213 OpcList: {amdgcn_cvt_scalef32_pk16_bf6_f16, amdgcn_cvt_scalef32_pk16_fp6_f16,
2214 amdgcn_cvt_scalef32_pk16_bf6_bf16, amdgcn_cvt_scalef32_pk16_fp6_bf16},
2215 FastTypes: Standard)
2216 .Any(Init: {.Predicate: {DivV3S32}, .OperandMapping: {{VgprV3S32}, {IntrId, VgprV16S16, Vgpr32}}})
2217 .Any(Init: {.Predicate: {UniV3S32}, .OperandMapping: {{UniInVgprV3S32}, {IntrId, VgprV16S16, Vgpr32}}});
2218
2219 addRulesForIOpcs(
2220 OpcList: {amdgcn_cvt_scalef32_pk16_bf6_f32, amdgcn_cvt_scalef32_pk16_fp6_f32},
2221 FastTypes: Standard)
2222 .Any(Init: {.Predicate: {DivV3S32}, .OperandMapping: {{VgprV3S32}, {IntrId, VgprV16S32, Vgpr32}}})
2223 .Any(Init: {.Predicate: {UniV3S32}, .OperandMapping: {{UniInVgprV3S32}, {IntrId, VgprV16S32, Vgpr32}}});
2224
2225 addRulesForIOpcs(
2226 OpcList: {amdgcn_cvt_scalef32_pk8_bf8_f16, amdgcn_cvt_scalef32_pk8_fp8_f16,
2227 amdgcn_cvt_scalef32_pk8_bf8_bf16, amdgcn_cvt_scalef32_pk8_fp8_bf16},
2228 FastTypes: Standard)
2229 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {IntrId, VgprV8S16, Vgpr32}}})
2230 .Any(Init: {.Predicate: {UniV2S32}, .OperandMapping: {{UniInVgprV2S32}, {IntrId, VgprV8S16, Vgpr32}}});
2231
2232 addRulesForIOpcs(
2233 OpcList: {amdgcn_cvt_scalef32_pk8_bf8_f32, amdgcn_cvt_scalef32_pk8_fp8_f32},
2234 FastTypes: Standard)
2235 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {IntrId, VgprV8S32, Vgpr32}}})
2236 .Any(Init: {.Predicate: {UniV2S32}, .OperandMapping: {{UniInVgprV2S32}, {IntrId, VgprV8S32, Vgpr32}}});
2237
2238 addRulesForIOpcs(
2239 OpcList: {amdgcn_cvt_scalef32_pk8_fp4_f16, amdgcn_cvt_scalef32_pk8_fp4_bf16},
2240 FastTypes: Standard)
2241 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, VgprV8S16, Vgpr32}})
2242 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, VgprV8S16, Vgpr32}});
2243
2244 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_pk8_fp4_f32}, FastTypes: Standard)
2245 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, VgprV8S32, Vgpr32}})
2246 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, VgprV8S32, Vgpr32}});
2247
2248 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_sr_pk16_bf6_f16,
2249 amdgcn_cvt_scalef32_sr_pk16_fp6_f16,
2250 amdgcn_cvt_scalef32_sr_pk16_bf6_bf16,
2251 amdgcn_cvt_scalef32_sr_pk16_fp6_bf16},
2252 FastTypes: Standard)
2253 .Any(Init: {.Predicate: {DivV3S32}, .OperandMapping: {{VgprV3S32}, {IntrId, VgprV16S16, Vgpr32, Vgpr32}}})
2254 .Any(Init: {.Predicate: {UniV3S32},
2255 .OperandMapping: {{UniInVgprV3S32}, {IntrId, VgprV16S16, Vgpr32, Vgpr32}}});
2256
2257 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_sr_pk16_bf6_f32,
2258 amdgcn_cvt_scalef32_sr_pk16_fp6_f32},
2259 FastTypes: Standard)
2260 .Any(Init: {.Predicate: {DivV3S32}, .OperandMapping: {{VgprV3S32}, {IntrId, VgprV16S32, Vgpr32, Vgpr32}}})
2261 .Any(Init: {.Predicate: {UniV3S32},
2262 .OperandMapping: {{UniInVgprV3S32}, {IntrId, VgprV16S32, Vgpr32, Vgpr32}}});
2263
2264 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_sr_pk8_bf8_f16,
2265 amdgcn_cvt_scalef32_sr_pk8_fp8_f16,
2266 amdgcn_cvt_scalef32_sr_pk8_bf8_bf16,
2267 amdgcn_cvt_scalef32_sr_pk8_fp8_bf16},
2268 FastTypes: Standard)
2269 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {IntrId, VgprV8S16, Vgpr32, Vgpr32}}})
2270 .Any(Init: {.Predicate: {UniV2S32},
2271 .OperandMapping: {{UniInVgprV2S32}, {IntrId, VgprV8S16, Vgpr32, Vgpr32}}});
2272
2273 addRulesForIOpcs(
2274 OpcList: {amdgcn_cvt_scalef32_sr_pk8_bf8_f32, amdgcn_cvt_scalef32_sr_pk8_fp8_f32},
2275 FastTypes: Standard)
2276 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {IntrId, VgprV8S32, Vgpr32, Vgpr32}}})
2277 .Any(Init: {.Predicate: {UniV2S32},
2278 .OperandMapping: {{UniInVgprV2S32}, {IntrId, VgprV8S32, Vgpr32, Vgpr32}}});
2279
2280 addRulesForIOpcs(
2281 OpcList: {amdgcn_cvt_scalef32_sr_pk8_fp4_f16, amdgcn_cvt_scalef32_sr_pk8_fp4_bf16},
2282 FastTypes: Standard)
2283 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, VgprV8S16, Vgpr32, Vgpr32}})
2284 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, VgprV8S16, Vgpr32, Vgpr32}});
2285
2286 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_sr_pk8_fp4_f32}, FastTypes: Standard)
2287 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, VgprV8S32, Vgpr32, Vgpr32}})
2288 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, VgprV8S32, Vgpr32, Vgpr32}});
2289
2290 addRulesForIOpcs(
2291 OpcList: {amdgcn_cvt_scale_pk16_f16_bf6, amdgcn_cvt_scale_pk16_f16_fp6,
2292 amdgcn_cvt_scale_pk16_bf16_bf6, amdgcn_cvt_scale_pk16_bf16_fp6},
2293 FastTypes: Standard)
2294 .Any(Init: {.Predicate: {DivV16S16}, .OperandMapping: {{VgprV16S16}, {IntrId, VgprV3S32, Vgpr32}}})
2295 .Any(Init: {.Predicate: {UniV16S16}, .OperandMapping: {{UniInVgprV16S16}, {IntrId, VgprV3S32, Vgpr32}}});
2296
2297 addRulesForIOpcs(
2298 OpcList: {amdgcn_cvt_scale_pk16_f32_bf6, amdgcn_cvt_scale_pk16_f32_fp6}, FastTypes: Standard)
2299 .Any(Init: {.Predicate: {DivV16S32}, .OperandMapping: {{VgprV16S32}, {IntrId, VgprV3S32, Vgpr32}}})
2300 .Any(Init: {.Predicate: {UniV16S32}, .OperandMapping: {{UniInVgprV16S32}, {IntrId, VgprV3S32, Vgpr32}}});
2301
2302 addRulesForIOpcs(
2303 OpcList: {amdgcn_cvt_scale_pk32_f16_bf6, amdgcn_cvt_scale_pk32_f16_fp6,
2304 amdgcn_cvt_scale_pk32_bf16_bf6, amdgcn_cvt_scale_pk32_bf16_fp6},
2305 FastTypes: Standard)
2306 .Any(Init: {.Predicate: {DivV32S16}, .OperandMapping: {{VgprV32S16}, {IntrId, VgprV6S32, Vgpr32}}})
2307 .Any(Init: {.Predicate: {UniV32S16}, .OperandMapping: {{UniInVgprV32S16}, {IntrId, VgprV6S32, Vgpr32}}});
2308
2309 addRulesForIOpcs(
2310 OpcList: {amdgcn_cvt_scale_pk32_f32_bf6, amdgcn_cvt_scale_pk32_f32_fp6}, FastTypes: Standard)
2311 .Any(Init: {.Predicate: {DivV32S32}, .OperandMapping: {{VgprV32S32}, {IntrId, VgprV6S32, Vgpr32}}})
2312 .Any(Init: {.Predicate: {UniV32S32}, .OperandMapping: {{UniInVgprV32S32}, {IntrId, VgprV6S32, Vgpr32}}});
2313
2314 addRulesForIOpcs(OpcList: {amdgcn_cvt_scale_pk8_f16_bf8, amdgcn_cvt_scale_pk8_f16_fp8,
2315 amdgcn_cvt_scale_pk8_bf16_bf8,
2316 amdgcn_cvt_scale_pk8_bf16_fp8},
2317 FastTypes: Standard)
2318 .Any(Init: {.Predicate: {DivV8S16}, .OperandMapping: {{VgprV8S16}, {IntrId, VgprV2S32, Vgpr32}}})
2319 .Any(Init: {.Predicate: {UniV8S16}, .OperandMapping: {{UniInVgprV8S16}, {IntrId, VgprV2S32, Vgpr32}}});
2320
2321 addRulesForIOpcs(
2322 OpcList: {amdgcn_cvt_scale_pk8_f16_fp4, amdgcn_cvt_scale_pk8_bf16_fp4}, FastTypes: Standard)
2323 .Any(Init: {.Predicate: {DivV8S16}, .OperandMapping: {{VgprV8S16}, {IntrId, Vgpr32, Vgpr32}}})
2324 .Any(Init: {.Predicate: {UniV8S16}, .OperandMapping: {{UniInVgprV8S16}, {IntrId, Vgpr32, Vgpr32}}});
2325
2326 addRulesForIOpcs(OpcList: {amdgcn_cvt_scale_pk8_f32_bf8, amdgcn_cvt_scale_pk8_f32_fp8},
2327 FastTypes: Standard)
2328 .Any(Init: {.Predicate: {DivV8S32}, .OperandMapping: {{VgprV8S32}, {IntrId, VgprV2S32, Vgpr32}}})
2329 .Any(Init: {.Predicate: {UniV8S32}, .OperandMapping: {{UniInVgprV8S32}, {IntrId, VgprV2S32, Vgpr32}}});
2330
2331 addRulesForIOpcs(OpcList: {amdgcn_cvt_scale_pk8_f32_fp4}, FastTypes: Standard)
2332 .Any(Init: {.Predicate: {DivV8S32}, .OperandMapping: {{VgprV8S32}, {IntrId, Vgpr32, Vgpr32}}})
2333 .Any(Init: {.Predicate: {UniV8S32}, .OperandMapping: {{UniInVgprV8S32}, {IntrId, Vgpr32, Vgpr32}}});
2334
2335 addRulesForIOpcs(
2336 OpcList: {amdgcn_cvt_scalef32_pk32_bf6_f16, amdgcn_cvt_scalef32_pk32_fp6_f16,
2337 amdgcn_cvt_scalef32_pk32_bf6_bf16, amdgcn_cvt_scalef32_pk32_fp6_bf16},
2338 FastTypes: Standard)
2339 .Any(Init: {.Predicate: {DivV6S32}, .OperandMapping: {{VgprV6S32}, {IntrId, VgprV32S16, Vgpr32}}})
2340 .Any(Init: {.Predicate: {UniV6S32}, .OperandMapping: {{UniInVgprV6S32}, {IntrId, VgprV32S16, Vgpr32}}});
2341
2342 addRulesForIOpcs(
2343 OpcList: {amdgcn_cvt_scalef32_pk32_bf6_f32, amdgcn_cvt_scalef32_pk32_fp6_f32},
2344 FastTypes: Standard)
2345 .Any(Init: {.Predicate: {DivV6S32}, .OperandMapping: {{VgprV6S32}, {IntrId, VgprV32S32, Vgpr32}}})
2346 .Any(Init: {.Predicate: {UniV6S32}, .OperandMapping: {{UniInVgprV6S32}, {IntrId, VgprV32S32, Vgpr32}}});
2347
2348 addRulesForIOpcs(
2349 OpcList: {amdgcn_cvt_scalef32_pk_fp8_f32, amdgcn_cvt_scalef32_pk_bf8_f32},
2350 FastTypes: Standard)
2351 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {IntrId, VgprV2S16, Vgpr32, Vgpr32, Vgpr32}})
2352 .Uni(Ty: V2S16,
2353 RuleApplyIDs: {{UniInVgprV2S16}, {IntrId, VgprV2S16, Vgpr32, Vgpr32, Vgpr32}});
2354
2355 addRulesForIOpcs(
2356 OpcList: {amdgcn_cvt_scalef32_pk_f32_fp8, amdgcn_cvt_scalef32_pk_f32_bf8},
2357 FastTypes: Standard)
2358 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {IntrId, Vgpr32, Vgpr32}}})
2359 .Any(Init: {.Predicate: {UniV2S32}, .OperandMapping: {{UniInVgprV2S32}, {IntrId, Vgpr32, Vgpr32}}});
2360
2361 addRulesForIOpcs(
2362 OpcList: {amdgcn_cvt_scalef32_pk_fp8_f16, amdgcn_cvt_scalef32_pk_bf8_f16,
2363 amdgcn_cvt_scalef32_pk_fp8_bf16, amdgcn_cvt_scalef32_pk_bf8_bf16},
2364 FastTypes: Standard)
2365 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {IntrId, VgprV2S16, VgprV2S16, Vgpr32}})
2366 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {IntrId, VgprV2S16, VgprV2S16, Vgpr32}});
2367
2368 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_pk_f32_fp4}, FastTypes: Standard)
2369 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {IntrId, Vgpr32, Vgpr32}}})
2370 .Any(Init: {.Predicate: {UniV2S32}, .OperandMapping: {{UniInVgprV2S32}, {IntrId, Vgpr32, Vgpr32}}});
2371
2372 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_pk_fp4_f32}, FastTypes: Standard)
2373 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32, Vgpr32}})
2374 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32, Vgpr32}});
2375
2376 addRulesForIOpcs(
2377 OpcList: {amdgcn_cvt_scalef32_pk_f16_fp4, amdgcn_cvt_scalef32_pk_f16_fp8,
2378 amdgcn_cvt_scalef32_pk_f16_bf8, amdgcn_cvt_scalef32_pk_bf16_fp4,
2379 amdgcn_cvt_scalef32_pk_bf16_fp8, amdgcn_cvt_scalef32_pk_bf16_bf8},
2380 FastTypes: Standard)
2381 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {IntrId, Vgpr32, Vgpr32}})
2382 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {IntrId, Vgpr32, Vgpr32}});
2383
2384 addRulesForIOpcs(
2385 OpcList: {amdgcn_cvt_scalef32_pk32_f32_fp6, amdgcn_cvt_scalef32_pk32_f32_bf6},
2386 FastTypes: Standard)
2387 .Any(Init: {.Predicate: {DivV32S32}, .OperandMapping: {{VgprV32S32}, {IntrId, VgprV6S32, Vgpr32}}})
2388 .Any(Init: {.Predicate: {UniV32S32}, .OperandMapping: {{UniInVgprV32S32}, {IntrId, VgprV6S32, Vgpr32}}});
2389
2390 addRulesForIOpcs(
2391 OpcList: {amdgcn_cvt_scalef32_pk32_f16_fp6, amdgcn_cvt_scalef32_pk32_f16_bf6,
2392 amdgcn_cvt_scalef32_pk32_bf16_fp6, amdgcn_cvt_scalef32_pk32_bf16_bf6},
2393 FastTypes: Standard)
2394 .Any(Init: {.Predicate: {DivV32S16}, .OperandMapping: {{VgprV32S16}, {IntrId, VgprV6S32, Vgpr32}}})
2395 .Any(Init: {.Predicate: {UniV32S16}, .OperandMapping: {{UniInVgprV32S16}, {IntrId, VgprV6S32, Vgpr32}}});
2396
2397 addRulesForIOpcs(
2398 OpcList: {amdgcn_cvt_scalef32_pk_fp4_f16, amdgcn_cvt_scalef32_pk_fp4_bf16},
2399 FastTypes: Standard)
2400 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, VgprV2S16, Vgpr32}})
2401 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, VgprV2S16, Vgpr32}});
2402
2403 addRulesForIOpcs(OpcList: {amdgcn_global_load_tr_b64})
2404 .Any(Init: {.Predicate: {DivB64, _, UniP1}, .OperandMapping: {{VgprB64}, {IntrId, SgprP1}}})
2405 .Any(Init: {.Predicate: {DivB64, _, DivP1}, .OperandMapping: {{VgprB64}, {IntrId, VgprP1}}})
2406 .Any(Init: {.Predicate: {DivB32, _, UniP1}, .OperandMapping: {{VgprB32}, {IntrId, SgprP1}}})
2407 .Any(Init: {.Predicate: {DivB32, _, DivP1}, .OperandMapping: {{VgprB32}, {IntrId, VgprP1}}});
2408
2409 addRulesForIOpcs(OpcList: {amdgcn_global_load_tr_b128})
2410 .Any(Init: {.Predicate: {DivB64, _, UniP1}, .OperandMapping: {{VgprB64}, {IntrId, SgprP1}}})
2411 .Any(Init: {.Predicate: {DivB64, _, DivP1}, .OperandMapping: {{VgprB64}, {IntrId, VgprP1}}})
2412 .Any(Init: {.Predicate: {DivB128, _, UniP1}, .OperandMapping: {{VgprB128}, {IntrId, SgprP1}}})
2413 .Any(Init: {.Predicate: {DivB128, _, DivP1}, .OperandMapping: {{VgprB128}, {IntrId, VgprP1}}});
2414
2415 addRulesForIOpcs(OpcList: {amdgcn_global_load_tr4_b64})
2416 .Any(Init: {.Predicate: {DivV2S32, _, UniP1}, .OperandMapping: {{VgprV2S32}, {IntrId, SgprP1}}})
2417 .Any(Init: {.Predicate: {DivV2S32, _, DivP1}, .OperandMapping: {{VgprV2S32}, {IntrId, VgprP1}}});
2418
2419 addRulesForIOpcs(OpcList: {amdgcn_global_load_tr6_b96})
2420 .Any(Init: {.Predicate: {DivV3S32, _, UniP1}, .OperandMapping: {{VgprV3S32}, {IntrId, SgprP1}}})
2421 .Any(Init: {.Predicate: {DivV3S32, _, DivP1}, .OperandMapping: {{VgprV3S32}, {IntrId, VgprP1}}});
2422
2423 addRulesForIOpcs(OpcList: {amdgcn_ds_load_tr4_b64, amdgcn_ds_load_tr8_b64})
2424 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {IntrId, VgprP3}}});
2425
2426 addRulesForIOpcs(OpcList: {amdgcn_ds_load_tr6_b96})
2427 .Any(Init: {.Predicate: {DivV3S32}, .OperandMapping: {{VgprV3S32}, {IntrId, VgprP3}}});
2428
2429 addRulesForIOpcs(OpcList: {amdgcn_ds_load_tr16_b128})
2430 .Any(Init: {.Predicate: {DivB128}, .OperandMapping: {{VgprB128}, {IntrId, VgprP3}}});
2431
2432 addRulesForIOpcs(OpcList: {amdgcn_global_atomic_ordered_add_b64})
2433 .Any(Init: {.Predicate: {DivS64}, .OperandMapping: {{Vgpr64}, {IntrId, VgprP1, Vgpr64}}});
2434
2435 addRulesForIOpcs(
2436 OpcList: {amdgcn_global_atomic_fmin_num, amdgcn_global_atomic_fmax_num}, FastTypes: Standard)
2437 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, VgprP1, Vgpr32}});
2438
2439 addRulesForIOpcs(OpcList: {amdgcn_flat_atomic_fmin_num, amdgcn_flat_atomic_fmax_num},
2440 FastTypes: Standard)
2441 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, VgprP0, Vgpr32}});
2442
2443 addRulesForIOpcs(OpcList: {amdgcn_raw_buffer_load_lds})
2444 .Any(Init: {.Predicate: {_}, .OperandMapping: {{}, {IntrId, SgprV4S32, SgprP3, Imm, Vgpr32, Sgpr32}}});
2445
2446 addRulesForIOpcs(OpcList: {amdgcn_raw_buffer_load_async_lds})
2447 .Any(Init: {.Predicate: {_}, .OperandMapping: {{}, {IntrId, SgprV4S32, SgprB32_M0, Imm, Vgpr32, Sgpr32}}});
2448
2449 addRulesForIOpcs(OpcList: {amdgcn_struct_buffer_load_async_lds})
2450 .Any(
2451 Init: {.Predicate: {_},
2452 .OperandMapping: {{}, {IntrId, SgprV4S32, SgprB32_M0, Imm, Vgpr32, Vgpr32, Sgpr32}}});
2453
2454 addRulesForIOpcs(OpcList: {amdgcn_struct_buffer_load_lds})
2455 .Any(Init: {.Predicate: {_},
2456 .OperandMapping: {{}, {IntrId, SgprV4S32, SgprP3, Imm, Vgpr32, Vgpr32, Sgpr32}}});
2457
2458 addRulesForIOpcs(OpcList: {amdgcn_raw_ptr_buffer_load_lds})
2459 .Any(Init: {.Predicate: {_}, .OperandMapping: {{}, {IntrId, SgprP8, SgprP3, Imm, Vgpr32, Sgpr32}}});
2460
2461 addRulesForIOpcs(OpcList: {amdgcn_raw_ptr_buffer_load_async_lds})
2462 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, SgprP8, SgprB32_M0, Imm, VgprB32, SgprB32}}});
2463
2464 addRulesForIOpcs(OpcList: {amdgcn_struct_ptr_buffer_load_async_lds})
2465 .Any(Init: {.Predicate: {_},
2466 .OperandMapping: {{}, {IntrId, SgprP8, SgprB32_M0, Imm, Vgpr32, Vgpr32, Sgpr32}}});
2467
2468 addRulesForIOpcs(OpcList: {amdgcn_struct_ptr_buffer_load_lds})
2469 .Any(Init: {.Predicate: {_}, .OperandMapping: {{}, {IntrId, SgprP8, SgprP3, Imm, Vgpr32, Vgpr32, Sgpr32}}});
2470
2471 addRulesForIOpcs(
2472 OpcList: {amdgcn_global_load_lds, amdgcn_load_to_lds, amdgcn_load_async_to_lds})
2473 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, VgprP1, SgprB32_M0}}});
2474
2475 addRulesForIOpcs(OpcList: {amdgcn_global_load_async_to_lds_b8,
2476 amdgcn_global_load_async_to_lds_b32,
2477 amdgcn_global_load_async_to_lds_b64,
2478 amdgcn_global_load_async_to_lds_b128,
2479 amdgcn_global_store_async_from_lds_b8,
2480 amdgcn_global_store_async_from_lds_b32,
2481 amdgcn_global_store_async_from_lds_b64,
2482 amdgcn_global_store_async_from_lds_b128})
2483 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, VgprP1, VgprP3}}});
2484
2485 addRulesForIOpcs(OpcList: {amdgcn_global_load_async_lds})
2486 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, VgprP1, SgprB32_M0}}});
2487
2488 addRulesForIOpcs(OpcList: {amdgcn_tensor_load_to_lds, amdgcn_tensor_store_from_lds})
2489 .Any(Init: {.Predicate: {},
2490 .OperandMapping: {{},
2491 {IntrId, SgprV4S32_ReadFirstLane, SgprV8S32_ReadFirstLane,
2492 SgprV4S32_ReadFirstLane, SgprV4S32_ReadFirstLane,
2493 SgprV8S32_ReadFirstLane}}});
2494
2495 addRulesForIOpcs(OpcList: {amdgcn_cluster_load_b32})
2496 .Any(Init: {.Predicate: {UniB32}, .OperandMapping: {{UniInVgprB32}, {IntrId, SgprP1, Imm, SgprB32_M0}}})
2497 .Any(Init: {.Predicate: {DivB32, _, UniP1}, .OperandMapping: {{VgprB32}, {IntrId, SgprP1, Imm, SgprB32_M0}}})
2498 .Any(
2499 Init: {.Predicate: {DivB32, _, DivP1}, .OperandMapping: {{VgprB32}, {IntrId, VgprP1, Imm, SgprB32_M0}}});
2500
2501 addRulesForIOpcs(OpcList: {amdgcn_cluster_load_b64})
2502 .Any(Init: {.Predicate: {UniB64}, .OperandMapping: {{UniInVgprB64}, {IntrId, SgprP1, Imm, SgprB32_M0}}})
2503 .Any(Init: {.Predicate: {DivB64, _, UniP1}, .OperandMapping: {{VgprB64}, {IntrId, SgprP1, Imm, SgprB32_M0}}})
2504 .Any(
2505 Init: {.Predicate: {DivB64, _, DivP1}, .OperandMapping: {{VgprB64}, {IntrId, VgprP1, Imm, SgprB32_M0}}});
2506
2507 addRulesForIOpcs(OpcList: {amdgcn_cluster_load_b128})
2508 .Any(Init: {.Predicate: {UniB128}, .OperandMapping: {{UniInVgprB128}, {IntrId, SgprP1, Imm, SgprB32_M0}}})
2509 .Any(Init: {.Predicate: {DivB128, _, UniP1},
2510 .OperandMapping: {{VgprB128}, {IntrId, SgprP1, Imm, SgprB32_M0}}})
2511 .Any(Init: {.Predicate: {DivB128, _, DivP1},
2512 .OperandMapping: {{VgprB128}, {IntrId, VgprP1, Imm, SgprB32_M0}}});
2513
2514 addRulesForIOpcs(OpcList: {amdgcn_cluster_load_async_to_lds_b8,
2515 amdgcn_cluster_load_async_to_lds_b32,
2516 amdgcn_cluster_load_async_to_lds_b64,
2517 amdgcn_cluster_load_async_to_lds_b128})
2518 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, VgprP1, VgprP3, Imm, Imm, SgprB32_M0}}});
2519
2520 addRulesForIOpcs(OpcList: {amdgcn_perm_pk16_b4_u4}, FastTypes: StandardB)
2521 .Uni(Ty: B64, RuleApplyIDs: {{UniInVgprB64}, {IntrId, Vgpr32, Vgpr32, VgprV2S32}})
2522 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {IntrId, Vgpr32, Vgpr32, VgprV2S32}});
2523
2524 addRulesForIOpcs(OpcList: {amdgcn_perm_pk16_b6_u4}, FastTypes: StandardB)
2525 .Uni(Ty: B96, RuleApplyIDs: {{UniInVgprB96}, {IntrId, Vgpr32, VgprB64, VgprV2S32}})
2526 .Div(Ty: B96, RuleApplyIDs: {{VgprB96}, {IntrId, Vgpr32, VgprB64, VgprV2S32}});
2527
2528 addRulesForIOpcs(OpcList: {amdgcn_perm_pk16_b8_u4}, FastTypes: StandardB)
2529 .Uni(Ty: B128, RuleApplyIDs: {{UniInVgprB128}, {IntrId, VgprB64, VgprB64, VgprV2S32}})
2530 .Div(Ty: B128, RuleApplyIDs: {{VgprB128}, {IntrId, VgprB64, VgprB64, VgprV2S32}});
2531
2532 addRulesForIOpcs(OpcList: {amdgcn_wwm, amdgcn_strict_wwm, amdgcn_wqm, amdgcn_softwqm,
2533 amdgcn_strict_wqm},
2534 FastTypes: StandardB)
2535 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {IntrId, VgprB32}})
2536 .Uni(Ty: B32, RuleApplyIDs: {{SgprB32}, {IntrId, SgprB32}})
2537 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {IntrId, VgprB64}})
2538 .Uni(Ty: B64, RuleApplyIDs: {{SgprB64}, {IntrId, SgprB64}})
2539 .Div(Ty: B96, RuleApplyIDs: {{VgprB96}, {IntrId, VgprB96}})
2540 .Uni(Ty: B96, RuleApplyIDs: {{SgprB96}, {IntrId, SgprB96}})
2541 .Div(Ty: B128, RuleApplyIDs: {{VgprB128}, {IntrId, VgprB128}})
2542 .Uni(Ty: B128, RuleApplyIDs: {{SgprB128}, {IntrId, SgprB128}})
2543 .Any(Init: {.Predicate: {UniB256}, .OperandMapping: {{SgprB256}, {IntrId, SgprB256}}})
2544 .Any(Init: {.Predicate: {DivB256}, .OperandMapping: {{VgprB256}, {IntrId, VgprB256}}})
2545 .Any(Init: {.Predicate: {UniB512}, .OperandMapping: {{SgprB512}, {IntrId, SgprB512}}})
2546 .Any(Init: {.Predicate: {DivB512}, .OperandMapping: {{VgprB512}, {IntrId, VgprB512}}});
2547
2548 addRulesForIOpcs(OpcList: {amdgcn_init_whole_wave}).Any(Init: {.Predicate: {DivS1}, .OperandMapping: {{Vcc}, {IntrId}}});
2549
2550 addRulesForIOpcs(OpcList: {amdgcn_kill, amdgcn_wqm_demote})
2551 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, Vcc}}});
2552
2553 addRulesForIOpcs(OpcList: {amdgcn_set_inactive}, FastTypes: StandardB)
2554 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {IntrId, VgprB32, VgprB32}});
2555
2556 addRulesForIOpcs(OpcList: {amdgcn_set_inactive_chain_arg}, FastTypes: Standard)
2557 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32}});
2558
2559 addRulesForIOpcs(OpcList: {amdgcn_cvt_sr_bf16_f32, amdgcn_cvt_sr_f16_f32}, FastTypes: Standard)
2560 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {IntrId, VgprV2S16, Vgpr32, Vgpr32}});
2561
2562 addRulesForIOpcs(OpcList: {amdgcn_ballot}, FastTypes: Standard)
2563 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {IntrId, Vcc}})
2564 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {IntrId, Vcc}});
2565
2566 addRulesForIOpcs(OpcList: {amdgcn_inverse_ballot})
2567 .Any(Init: {.Predicate: {DivS1, _, S32}, .OperandMapping: {{Vcc}, {IntrId, SgprB32_ReadFirstLane}}})
2568 .Any(Init: {.Predicate: {DivS1, _, S64}, .OperandMapping: {{Vcc}, {IntrId, SgprB64_ReadFirstLane}}});
2569
2570 addRulesForIOpcs(OpcList: {amdgcn_live_mask, amdgcn_ps_live})
2571 .Any(Init: {.Predicate: {DivS1}, .OperandMapping: {{Vcc}, {}}});
2572
2573 addRulesForIOpcs(OpcList: {amdgcn_mov_dpp, amdgcn_mov_dpp8}, FastTypes: StandardB)
2574 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {IntrId, VgprB32}})
2575 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {IntrId, VgprB64}});
2576
2577 addRulesForIOpcs(OpcList: {amdgcn_update_dpp}, FastTypes: StandardB)
2578 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {IntrId, VgprB32, VgprB32}})
2579 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {IntrId, VgprB64, VgprB64}});
2580
2581 addRulesForIOpcs(OpcList: {amdgcn_sin, amdgcn_cos, amdgcn_tanh}, FastTypes: Standard)
2582 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, Vgpr16}})
2583 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, Vgpr16}})
2584 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32}})
2585 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32}});
2586
2587 addRulesForIOpcs(OpcList: {amdgcn_trig_preop}, FastTypes: Standard)
2588 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {IntrId, Vgpr64, Vgpr32}})
2589 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {IntrId, Vgpr64, Vgpr32}});
2590
2591 addRulesForIOpcs(OpcList: {amdgcn_exp2})
2592 .Any(Init: {.Predicate: {DivS16}, .OperandMapping: {{Vgpr16}, {IntrId, Vgpr16}}})
2593 .Any(Init: {.Predicate: {UniBF16}, .OperandMapping: {{UniInVgprS16}, {IntrId, Vgpr16}}})
2594 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{Sgpr16}, {IntrId, Sgpr16}}}, STPred: hasPST)
2595 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{UniInVgprS16}, {IntrId, Vgpr16}}}, STPred: !hasPST)
2596 .Any(Init: {.Predicate: {DivS32}, .OperandMapping: {{Vgpr32}, {IntrId, Vgpr32}}})
2597 .Any(Init: {.Predicate: {UniS32}, .OperandMapping: {{Sgpr32}, {IntrId, Sgpr32}}}, STPred: hasPST)
2598 .Any(Init: {.Predicate: {UniS32}, .OperandMapping: {{UniInVgprS32}, {IntrId, Vgpr32}}}, STPred: !hasPST);
2599
2600 addRulesForIOpcs(OpcList: {amdgcn_rcp, amdgcn_sqrt})
2601 .Any(Init: {.Predicate: {DivS16}, .OperandMapping: {{Vgpr16}, {IntrId, Vgpr16}}})
2602 .Any(Init: {.Predicate: {UniBF16}, .OperandMapping: {{UniInVgprS16}, {IntrId, Vgpr16}}})
2603 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{Sgpr16}, {IntrId, Sgpr16}}}, STPred: hasPST)
2604 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{UniInVgprS16}, {IntrId, Vgpr16}}}, STPred: !hasPST)
2605 .Any(Init: {.Predicate: {DivS32}, .OperandMapping: {{Vgpr32}, {IntrId, Vgpr32}}})
2606 .Any(Init: {.Predicate: {UniS32}, .OperandMapping: {{Sgpr32}, {IntrId, Sgpr32}}}, STPred: hasPST)
2607 .Any(Init: {.Predicate: {UniS32}, .OperandMapping: {{UniInVgprS32}, {IntrId, Vgpr32}}}, STPred: !hasPST)
2608 .Any(Init: {.Predicate: {DivS64}, .OperandMapping: {{Vgpr64}, {IntrId, Vgpr64}}})
2609 .Any(Init: {.Predicate: {UniS64}, .OperandMapping: {{UniInVgprS64}, {IntrId, Vgpr64}}});
2610
2611 addRulesForIOpcs(OpcList: {amdgcn_log})
2612 .Any(Init: {.Predicate: {DivS16}, .OperandMapping: {{Vgpr16}, {IntrId, Vgpr16}}})
2613 .Any(Init: {.Predicate: {UniBF16}, .OperandMapping: {{UniInVgprS16}, {IntrId, Vgpr16}}})
2614 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{Sgpr16}, {IntrId, Sgpr16}}}, STPred: hasPST)
2615 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{UniInVgprS16}, {IntrId, Vgpr16}}}, STPred: !hasPST)
2616 .Any(Init: {.Predicate: {DivS32}, .OperandMapping: {{Vgpr32}, {IntrId, Vgpr32}}})
2617 .Any(Init: {.Predicate: {UniS32}, .OperandMapping: {{Sgpr32}, {IntrId, Sgpr32}}}, STPred: hasPST)
2618 .Any(Init: {.Predicate: {UniS32}, .OperandMapping: {{UniInVgprS32}, {IntrId, Vgpr32}}}, STPred: !hasPST);
2619
2620 addRulesForIOpcs(OpcList: {amdgcn_ds_atomic_async_barrier_arrive_b64})
2621 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, VgprP3}}});
2622
2623 addRulesForIOpcs(OpcList: {amdgcn_ds_atomic_barrier_arrive_rtn_b64}, FastTypes: Standard)
2624 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {IntrId, VgprP3, Vgpr64}});
2625
2626 addRulesForIOpcs(OpcList: {amdgcn_ds_add_gs_reg_rtn, amdgcn_ds_sub_gs_reg_rtn},
2627 FastTypes: Standard)
2628 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32}})
2629 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {IntrId, Vgpr32}});
2630
2631 addRulesForIOpcs(OpcList: {amdgcn_ds_append, amdgcn_ds_consume}, FastTypes: Standard)
2632 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, SgprB32_M0}})
2633 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, SgprB32_M0}});
2634
2635 addRulesForIOpcs(
2636 OpcList: {amdgcn_ds_bvh_stack_rtn, amdgcn_ds_bvh_stack_push4_pop1_rtn}, FastTypes: Standard)
2637 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32, Vgpr32}, {IntrId, Vgpr32, Vgpr32, VgprV4S32}});
2638
2639 addRulesForIOpcs(OpcList: {amdgcn_ds_bvh_stack_push8_pop1_rtn}, FastTypes: Standard)
2640 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32, Vgpr32}, {IntrId, Vgpr32, Vgpr32, VgprV8S32}});
2641
2642 addRulesForIOpcs(OpcList: {amdgcn_ds_bvh_stack_push8_pop2_rtn}, FastTypes: Standard)
2643 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64, Vgpr32}, {IntrId, Vgpr32, Vgpr32, VgprV8S32}});
2644
2645 addRulesForIOpcs(OpcList: {amdgcn_ds_gws_sema_p, amdgcn_ds_gws_sema_v,
2646 amdgcn_ds_gws_sema_release_all})
2647 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, SgprB32_M0}}});
2648
2649 addRulesForIOpcs(
2650 OpcList: {amdgcn_ds_gws_barrier, amdgcn_ds_gws_init, amdgcn_ds_gws_sema_br})
2651 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, Vgpr32, SgprB32_M0}}});
2652
2653 addRulesForIOpcs(OpcList: {amdgcn_ds_ordered_add, amdgcn_ds_ordered_swap}, FastTypes: Standard)
2654 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, SgprB32_M0, Vgpr32}});
2655
2656 addRulesForIOpcs(OpcList: {amdgcn_ds_swizzle}, FastTypes: Standard)
2657 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32}})
2658 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32}});
2659
2660 addRulesForIOpcs(OpcList: {amdgcn_permlane16_var, amdgcn_permlanex16_var}, FastTypes: Standard)
2661 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
2662
2663 addRulesForIOpcs(OpcList: {amdgcn_permlane16_swap, amdgcn_permlane32_swap}, FastTypes: Standard)
2664 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32, Vgpr32}, {IntrId, Vgpr32, Vgpr32}});
2665
2666 addRulesForIOpcs(OpcList: {amdgcn_permlane64}, FastTypes: StandardB)
2667 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {IntrId, VgprB32}});
2668
2669 addRulesForIOpcs(OpcList: {amdgcn_ds_read_tr4_b64, amdgcn_ds_read_tr8_b64})
2670 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {IntrId, VgprP3}}});
2671
2672 addRulesForIOpcs(OpcList: {amdgcn_ds_read_tr6_b96})
2673 .Any(Init: {.Predicate: {DivV3S32}, .OperandMapping: {{VgprV3S32}, {IntrId, VgprP3}}});
2674
2675 addRulesForIOpcs(OpcList: {amdgcn_ds_read_tr16_b64})
2676 .Any(Init: {.Predicate: {DivV4S16}, .OperandMapping: {{VgprV4S16}, {IntrId, VgprP3}}});
2677
2678 addRulesForIOpcs(OpcList: {amdgcn_interp_p1}, FastTypes: Standard)
2679 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Imm, Imm, SgprB32_M0}});
2680
2681 addRulesForIOpcs(OpcList: {amdgcn_interp_p1_f16}, FastTypes: Standard)
2682 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Imm, Imm, Imm, SgprB32_M0}});
2683
2684 addRulesForIOpcs(OpcList: {amdgcn_interp_p2}, FastTypes: Standard)
2685 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Imm, Imm, SgprB32_M0}});
2686
2687 addRulesForIOpcs(OpcList: {amdgcn_interp_p2_f16}, FastTypes: Standard)
2688 .Div(Ty: S16,
2689 RuleApplyIDs: {{Vgpr16}, {IntrId, Vgpr32, Vgpr32, Imm, Imm, Imm, SgprB32_M0}});
2690
2691 addRulesForIOpcs(OpcList: {amdgcn_interp_mov}, FastTypes: Standard)
2692 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Imm, Imm, Imm, SgprB32_M0}});
2693
2694 addRulesForIOpcs(OpcList: {amdgcn_interp_inreg_p10, amdgcn_interp_inreg_p2,
2695 amdgcn_interp_inreg_p10_f16, amdgcn_interp_p10_rtz_f16},
2696 FastTypes: Standard)
2697 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
2698 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
2699
2700 addRulesForIOpcs(OpcList: {amdgcn_interp_inreg_p2_f16, amdgcn_interp_p2_rtz_f16},
2701 FastTypes: Standard)
2702 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
2703 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
2704
2705 addRulesForIOpcs(OpcList: {amdgcn_frexp_exp})
2706 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{UniInVgprS16}, {IntrId, Vgpr16}}})
2707 .Any(Init: {.Predicate: {DivS16}, .OperandMapping: {{Vgpr16}, {IntrId, Vgpr16}}})
2708 .Any(Init: {.Predicate: {UniS32, _, S32}, .OperandMapping: {{UniInVgprS32}, {IntrId, Vgpr32}}})
2709 .Any(Init: {.Predicate: {DivS32, _, S32}, .OperandMapping: {{Vgpr32}, {IntrId, Vgpr32}}})
2710 .Any(Init: {.Predicate: {UniS32, _, S64}, .OperandMapping: {{UniInVgprS32}, {IntrId, Vgpr64}}})
2711 .Any(Init: {.Predicate: {DivS32, _, S64}, .OperandMapping: {{Vgpr32}, {IntrId, Vgpr64}}});
2712
2713 Predicate isUnscaledDivFmas([](const MachineInstr &MI) -> bool {
2714 return AMDGPU::isUnscaledIEEEDivFmas(MI, MRI: MI.getMF()->getRegInfo());
2715 });
2716
2717 addRulesForIOpcs(OpcList: {amdgcn_div_fmas})
2718 .Any(Init: {.Predicate: {DivS32}, .OperandMapping: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32, Vcc}}})
2719 .Any(Init: {.Predicate: {{UniS32}, isUnscaledDivFmas},
2720 .OperandMapping: {{Sgpr32}, {IntrId, Sgpr32, Sgpr32, Sgpr32, Vcc}}},
2721 STPred: hasSALUFloat)
2722 .Any(Init: {.Predicate: {UniS32}, .OperandMapping: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32, Vcc}}})
2723 .Any(Init: {.Predicate: {DivS64}, .OperandMapping: {{Vgpr64}, {IntrId, Vgpr64, Vgpr64, Vgpr64, Vcc}}})
2724 .Any(Init: {.Predicate: {UniS64}, .OperandMapping: {{UniInVgprS64}, {IntrId, Vgpr64, Vgpr64, Vgpr64, Vcc}}});
2725
2726 addRulesForIOpcs(OpcList: {amdgcn_div_fixup}, FastTypes: Standard)
2727 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, Vgpr16, Vgpr16, Vgpr16}})
2728 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, Vgpr16, Vgpr16, Vgpr16}})
2729 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
2730 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
2731 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {IntrId, Vgpr64, Vgpr64, Vgpr64}})
2732 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {IntrId, Vgpr64, Vgpr64, Vgpr64}});
2733
2734 addRulesForIOpcs(OpcList: {amdgcn_div_scale}, FastTypes: Standard)
2735 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32, Vcc}, {IntrId, Vgpr32, Vgpr32}})
2736 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32, UniInVcc}, {IntrId, Vgpr32, Vgpr32}})
2737 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64, Vcc}, {IntrId, Vgpr64, Vgpr64}})
2738 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64, UniInVcc}, {IntrId, Vgpr64, Vgpr64}});
2739
2740 addRulesForIOpcs(
2741 OpcList: {amdgcn_fdot2, amdgcn_fdot2_f32_bf16, amdgcn_sdot2, amdgcn_udot2},
2742 FastTypes: Standard)
2743 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, VgprV2S16, VgprV2S16, Vgpr32}})
2744 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, VgprV2S16, VgprV2S16, Vgpr32}});
2745
2746 addRulesForIOpcs(OpcList: {amdgcn_fdot2_f16_f16, amdgcn_fdot2_bf16_bf16}, FastTypes: Standard)
2747 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, VgprV2S16, VgprV2S16, Vgpr16}})
2748 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, VgprV2S16, VgprV2S16, Vgpr16}});
2749
2750 addRulesForIOpcs(OpcList: {amdgcn_sudot4, amdgcn_sudot8}, FastTypes: Standard)
2751 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Imm, Vgpr32, Imm, Vgpr32, Vgpr32}})
2752 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Imm, Vgpr32, Imm, Vgpr32, Vgpr32}});
2753
2754 addRulesForIOpcs(OpcList: {amdgcn_s_alloc_vgpr})
2755 .Any(Init: {.Predicate: {UniS1}, .OperandMapping: {{Sgpr32Trunc}, {IntrId, SgprB32_ReadFirstLane}}});
2756
2757 addRulesForIOpcs(OpcList: {amdgcn_sat_pk4_i4_i8, amdgcn_sat_pk4_u4_u8}, FastTypes: Standard)
2758 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, Vgpr32}})
2759 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, Vgpr32}});
2760
2761 bool HasGFX90AInsts = ST->hasGFX90AInsts();
2762
2763 // On gfx90a+ both AGPR-form and VGPR-form exists
2764 addRulesForIOpcs(OpcList: {amdgcn_mfma_f32_32x32x1f32, amdgcn_mfma_f32_16x16x1f32,
2765 amdgcn_mfma_f32_4x4x1f32, amdgcn_mfma_f32_32x32x2f32,
2766 amdgcn_mfma_f32_16x16x4f32, amdgcn_mfma_f32_32x32x4f16,
2767 amdgcn_mfma_f32_16x16x4f16, amdgcn_mfma_f32_4x4x4f16,
2768 amdgcn_mfma_f32_32x32x8f16, amdgcn_mfma_f32_16x16x16f16,
2769 amdgcn_mfma_i32_32x32x4i8, amdgcn_mfma_i32_16x16x4i8,
2770 amdgcn_mfma_i32_4x4x4i8, amdgcn_mfma_i32_32x32x8i8,
2771 amdgcn_mfma_i32_16x16x16i8, amdgcn_mfma_f32_32x32x2bf16,
2772 amdgcn_mfma_f32_16x16x2bf16, amdgcn_mfma_f32_4x4x2bf16,
2773 amdgcn_mfma_f32_32x32x4bf16, amdgcn_mfma_f32_16x16x8bf16})
2774 .Any(Init: {.Predicate: {DivAnyTy},
2775 .OperandMapping: {{AgprAnyTy}, {IntrId, VgprAnyTy, VgprAnyTy, AgprAnyTy}}},
2776 STPred: !HasGFX90AInsts)
2777 .Any(Init: {.Predicate: {DivAnyTy},
2778 .OperandMapping: {{VgprOrAgprAnyTy},
2779 {IntrId, VgprAnyTy, VgprAnyTy, VgprOrAgprAnyTy}}},
2780 STPred: HasGFX90AInsts);
2781
2782 // gfx90a+ only MFMAs
2783 addRulesForIOpcs(OpcList: {
2784 amdgcn_mfma_f32_32x32x4bf16_1k,
2785 amdgcn_mfma_f32_16x16x4bf16_1k,
2786 amdgcn_mfma_f32_4x4x4bf16_1k,
2787 amdgcn_mfma_f32_32x32x8bf16_1k,
2788 amdgcn_mfma_f32_16x16x16bf16_1k,
2789 amdgcn_mfma_f64_16x16x4f64,
2790 amdgcn_mfma_f64_4x4x4f64,
2791 amdgcn_mfma_i32_16x16x32_i8,
2792 amdgcn_mfma_i32_32x32x16_i8,
2793 amdgcn_mfma_f32_16x16x8_xf32,
2794 amdgcn_mfma_f32_32x32x4_xf32,
2795 amdgcn_mfma_f32_16x16x32_bf8_bf8,
2796 amdgcn_mfma_f32_16x16x32_bf8_fp8,
2797 amdgcn_mfma_f32_16x16x32_fp8_bf8,
2798 amdgcn_mfma_f32_16x16x32_fp8_fp8,
2799 amdgcn_mfma_f32_32x32x16_bf8_bf8,
2800 amdgcn_mfma_f32_32x32x16_bf8_fp8,
2801 amdgcn_mfma_f32_32x32x16_fp8_bf8,
2802 amdgcn_mfma_f32_32x32x16_fp8_fp8,
2803 // gfx950
2804 amdgcn_mfma_f32_16x16x32_f16,
2805 amdgcn_mfma_f32_32x32x16_f16,
2806 amdgcn_mfma_i32_16x16x64_i8,
2807 amdgcn_mfma_i32_32x32x32_i8,
2808 amdgcn_mfma_f32_16x16x32_bf16,
2809 amdgcn_mfma_f32_32x32x16_bf16,
2810 })
2811 .Any(Init: {.Predicate: {DivAnyTy},
2812 .OperandMapping: {{VgprOrAgprAnyTy},
2813 {IntrId, VgprAnyTy, VgprAnyTy, VgprOrAgprAnyTy}}});
2814
2815 addRulesForIOpcs(
2816 OpcList: {// gfx942+
2817 amdgcn_smfmac_f32_16x16x32_f16, amdgcn_smfmac_f32_32x32x16_f16,
2818 amdgcn_smfmac_f32_16x16x32_bf16, amdgcn_smfmac_f32_32x32x16_bf16,
2819 amdgcn_smfmac_i32_16x16x64_i8, amdgcn_smfmac_i32_32x32x32_i8,
2820 amdgcn_smfmac_f32_16x16x64_bf8_bf8, amdgcn_smfmac_f32_16x16x64_bf8_fp8,
2821 amdgcn_smfmac_f32_16x16x64_fp8_bf8, amdgcn_smfmac_f32_16x16x64_fp8_fp8,
2822 amdgcn_smfmac_f32_32x32x32_bf8_bf8, amdgcn_smfmac_f32_32x32x32_bf8_fp8,
2823 amdgcn_smfmac_f32_32x32x32_fp8_bf8, amdgcn_smfmac_f32_32x32x32_fp8_fp8,
2824 // gfx950+
2825 amdgcn_smfmac_f32_16x16x64_f16, amdgcn_smfmac_f32_32x32x32_f16,
2826 amdgcn_smfmac_f32_16x16x64_bf16, amdgcn_smfmac_f32_32x32x32_bf16,
2827 amdgcn_smfmac_i32_16x16x128_i8, amdgcn_smfmac_i32_32x32x64_i8,
2828 amdgcn_smfmac_f32_16x16x128_bf8_bf8, amdgcn_smfmac_f32_16x16x128_bf8_fp8,
2829 amdgcn_smfmac_f32_16x16x128_fp8_bf8, amdgcn_smfmac_f32_16x16x128_fp8_fp8,
2830 amdgcn_smfmac_f32_32x32x64_bf8_bf8, amdgcn_smfmac_f32_32x32x64_bf8_fp8,
2831 amdgcn_smfmac_f32_32x32x64_fp8_bf8, amdgcn_smfmac_f32_32x32x64_fp8_fp8})
2832 .Any(Init: {.Predicate: {DivAnyTy},
2833 .OperandMapping: {{VgprOrAgprAnyTy},
2834 {IntrId, VgprAnyTy, VgprAnyTy, VgprOrAgprAnyTy, VgprAnyTy}}});
2835
2836 addRulesForIOpcs(OpcList: {amdgcn_mfma_scale_f32_32x32x64_f8f6f4,
2837 amdgcn_mfma_scale_f32_16x16x128_f8f6f4})
2838 .Any(Init: {.Predicate: {DivAnyTy},
2839 .OperandMapping: {{VgprOrAgprAnyTy},
2840 {IntrId, VgprAnyTy, VgprAnyTy, VgprOrAgprAnyTy, Imm, Imm, Imm,
2841 Vgpr32, Imm, Vgpr32}}});
2842
2843 // WMMA/SWMMAC intrinsics: all register operands map to VGPR.
2844 addRulesForIOpcs(
2845 OpcList: {// WMMA GFX11+
2846 amdgcn_wmma_f32_16x16x16_f16, amdgcn_wmma_f32_16x16x16_bf16,
2847 amdgcn_wmma_f16_16x16x16_f16, amdgcn_wmma_bf16_16x16x16_bf16,
2848 amdgcn_wmma_f16_16x16x16_f16_tied, amdgcn_wmma_bf16_16x16x16_bf16_tied,
2849 amdgcn_wmma_i32_16x16x16_iu8, amdgcn_wmma_i32_16x16x16_iu4,
2850 // WMMA GFX12
2851 amdgcn_wmma_f32_16x16x16_fp8_fp8, amdgcn_wmma_f32_16x16x16_fp8_bf8,
2852 amdgcn_wmma_f32_16x16x16_bf8_fp8, amdgcn_wmma_f32_16x16x16_bf8_bf8,
2853 amdgcn_wmma_i32_16x16x32_iu4,
2854 // WMMA GFX1250
2855 amdgcn_wmma_f32_16x16x4_f32, amdgcn_wmma_f32_16x16x32_bf16,
2856 amdgcn_wmma_f32_16x16x32_f16, amdgcn_wmma_f16_16x16x32_f16,
2857 amdgcn_wmma_bf16_16x16x32_bf16, amdgcn_wmma_bf16f32_16x16x32_bf16,
2858 amdgcn_wmma_f32_16x16x64_fp8_fp8, amdgcn_wmma_f32_16x16x64_fp8_bf8,
2859 amdgcn_wmma_f32_16x16x64_bf8_fp8, amdgcn_wmma_f32_16x16x64_bf8_bf8,
2860 amdgcn_wmma_f16_16x16x64_fp8_fp8, amdgcn_wmma_f16_16x16x64_fp8_bf8,
2861 amdgcn_wmma_f16_16x16x64_bf8_fp8, amdgcn_wmma_f16_16x16x64_bf8_bf8,
2862 amdgcn_wmma_f16_16x16x128_fp8_fp8, amdgcn_wmma_f16_16x16x128_fp8_bf8,
2863 amdgcn_wmma_f16_16x16x128_bf8_fp8, amdgcn_wmma_f16_16x16x128_bf8_bf8,
2864 amdgcn_wmma_f32_16x16x128_fp8_fp8, amdgcn_wmma_f32_16x16x128_fp8_bf8,
2865 amdgcn_wmma_f32_16x16x128_bf8_fp8, amdgcn_wmma_f32_16x16x128_bf8_bf8,
2866 amdgcn_wmma_i32_16x16x64_iu8, amdgcn_wmma_f32_16x16x128_f8f6f4,
2867 amdgcn_wmma_scale_f32_16x16x128_f8f6f4,
2868 amdgcn_wmma_scale16_f32_16x16x128_f8f6f4, amdgcn_wmma_f32_32x16x128_f4,
2869 amdgcn_wmma_scale_f32_32x16x128_f4, amdgcn_wmma_scale16_f32_32x16x128_f4,
2870 // WMMA GFX1251
2871 amdgcn_wmma_f64_16x16x4_f64,
2872 // SWMMAC GFX12
2873 amdgcn_swmmac_f32_16x16x32_f16, amdgcn_swmmac_f32_16x16x32_bf16,
2874 amdgcn_swmmac_f16_16x16x32_f16, amdgcn_swmmac_bf16_16x16x32_bf16,
2875 amdgcn_swmmac_i32_16x16x32_iu8, amdgcn_swmmac_i32_16x16x32_iu4,
2876 amdgcn_swmmac_i32_16x16x64_iu4, amdgcn_swmmac_f32_16x16x32_fp8_fp8,
2877 amdgcn_swmmac_f32_16x16x32_fp8_bf8, amdgcn_swmmac_f32_16x16x32_bf8_fp8,
2878 amdgcn_swmmac_f32_16x16x32_bf8_bf8,
2879 // SWMMAC GFX1250
2880 amdgcn_swmmac_f32_16x16x64_f16, amdgcn_swmmac_f32_16x16x64_bf16,
2881 amdgcn_swmmac_f16_16x16x64_f16, amdgcn_swmmac_bf16_16x16x64_bf16,
2882 amdgcn_swmmac_bf16f32_16x16x64_bf16, amdgcn_swmmac_f32_16x16x128_fp8_fp8,
2883 amdgcn_swmmac_f32_16x16x128_fp8_bf8, amdgcn_swmmac_f32_16x16x128_bf8_fp8,
2884 amdgcn_swmmac_f32_16x16x128_bf8_bf8, amdgcn_swmmac_f16_16x16x128_fp8_fp8,
2885 amdgcn_swmmac_f16_16x16x128_fp8_bf8, amdgcn_swmmac_f16_16x16x128_bf8_fp8,
2886 amdgcn_swmmac_f16_16x16x128_bf8_bf8, amdgcn_swmmac_i32_16x16x128_iu8})
2887 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {}, ApplyAllVgpr}});
2888
2889} // end initialize rules
2890