1//===-- AMDGPURegBankLegalizeRules.cpp ------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// Definitions of RegBankLegalize Rules for all opcodes.
10/// Implementation of container for all the Rules and search.
11/// Fast search for most common case when Rule.Predicate checks LLT and
12/// uniformity of register in operand 0.
13//
14//===----------------------------------------------------------------------===//
15
16#include "AMDGPURegBankLegalizeRules.h"
17#include "AMDGPUInstrInfo.h"
18#include "GCNSubtarget.h"
19#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
20#include "llvm/CodeGen/MachineUniformityAnalysis.h"
21#include "llvm/IR/IntrinsicsAMDGPU.h"
22#include "llvm/Support/AMDGPUAddrSpace.h"
23
24#define DEBUG_TYPE "amdgpu-reg-bank-legalize"
25
26using namespace llvm;
27using namespace AMDGPU;
28
29bool AMDGPU::isAnyPtr(LLT Ty, unsigned Width) {
30 return Ty.isPointer() && Ty.getSizeInBits() == Width;
31}
32
33RegBankLLTMapping::RegBankLLTMapping(
34 std::initializer_list<RegBankLLTMappingApplyID> DstOpMappingList,
35 std::initializer_list<RegBankLLTMappingApplyID> SrcOpMappingList,
36 LoweringMethodID LoweringMethod)
37 : DstOpMapping(DstOpMappingList), SrcOpMapping(SrcOpMappingList),
38 LoweringMethod(LoweringMethod) {}
39
40PredicateMapping::PredicateMapping(
41 std::initializer_list<UniformityLLTOpPredicateID> OpList,
42 std::function<bool(const MachineInstr &)> TestFunc)
43 : OpUniformityAndTypes(OpList), TestFunc(TestFunc) {}
44
45bool matchUniformityAndLLT(Register Reg, UniformityLLTOpPredicateID UniID,
46 const MachineUniformityInfo &MUI,
47 const MachineRegisterInfo &MRI) {
48 switch (UniID) {
49 case S1:
50 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 1);
51 case S16:
52 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 16);
53 case S32:
54 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 32);
55 case S64:
56 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 64);
57 case S128:
58 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 128);
59 case P0:
60 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 0, SizeInBits: 64);
61 case P1:
62 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 1, SizeInBits: 64);
63 case P2:
64 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 2, SizeInBits: 32);
65 case P3:
66 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 3, SizeInBits: 32);
67 case P4:
68 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 4, SizeInBits: 64);
69 case P5:
70 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 5, SizeInBits: 32);
71 case P8:
72 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 8, SizeInBits: 128);
73 case Ptr32:
74 return isAnyPtr(Ty: MRI.getType(Reg), Width: 32);
75 case Ptr64:
76 return isAnyPtr(Ty: MRI.getType(Reg), Width: 64);
77 case Ptr128:
78 return isAnyPtr(Ty: MRI.getType(Reg), Width: 128);
79 case V2S16:
80 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16);
81 case V2S32:
82 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32);
83 case V3S32:
84 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 32);
85 case V4S32:
86 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32);
87 case B32:
88 return MRI.getType(Reg).getSizeInBits() == 32;
89 case B64:
90 return MRI.getType(Reg).getSizeInBits() == 64;
91 case B96:
92 return MRI.getType(Reg).getSizeInBits() == 96;
93 case B128:
94 return MRI.getType(Reg).getSizeInBits() == 128;
95 case B160:
96 return MRI.getType(Reg).getSizeInBits() == 160;
97 case B256:
98 return MRI.getType(Reg).getSizeInBits() == 256;
99 case B512:
100 return MRI.getType(Reg).getSizeInBits() == 512;
101 case DivAnyTy:
102 return MUI.isDivergentAtDef(V: Reg);
103 case UniS1:
104 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 1) && MUI.isUniformAtDef(V: Reg);
105 case UniS16:
106 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 16) && MUI.isUniformAtDef(V: Reg);
107 case UniS32:
108 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 32) && MUI.isUniformAtDef(V: Reg);
109 case UniS64:
110 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 64) && MUI.isUniformAtDef(V: Reg);
111 case UniS128:
112 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 128) && MUI.isUniformAtDef(V: Reg);
113 case UniBF16:
114 return MRI.getType(Reg).isBFloat16() && MUI.isUniformAtDef(V: Reg);
115 case UniP0:
116 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 0, SizeInBits: 64) && MUI.isUniformAtDef(V: Reg);
117 case UniP1:
118 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 1, SizeInBits: 64) && MUI.isUniformAtDef(V: Reg);
119 case UniP2:
120 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 2, SizeInBits: 32) && MUI.isUniformAtDef(V: Reg);
121 case UniP3:
122 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 3, SizeInBits: 32) && MUI.isUniformAtDef(V: Reg);
123 case UniP4:
124 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 4, SizeInBits: 64) && MUI.isUniformAtDef(V: Reg);
125 case UniP5:
126 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 5, SizeInBits: 32) && MUI.isUniformAtDef(V: Reg);
127 case UniP6:
128 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 6, SizeInBits: 32) && MUI.isUniformAtDef(V: Reg);
129 case UniP8:
130 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 8, SizeInBits: 128) && MUI.isUniformAtDef(V: Reg);
131 case UniPtr32:
132 return isAnyPtr(Ty: MRI.getType(Reg), Width: 32) && MUI.isUniformAtDef(V: Reg);
133 case UniPtr64:
134 return isAnyPtr(Ty: MRI.getType(Reg), Width: 64) && MUI.isUniformAtDef(V: Reg);
135 case UniPtr128:
136 return isAnyPtr(Ty: MRI.getType(Reg), Width: 128) && MUI.isUniformAtDef(V: Reg);
137 case UniV2S16:
138 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16) &&
139 MUI.isUniformAtDef(V: Reg);
140 case UniV2S32:
141 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32) &&
142 MUI.isUniformAtDef(V: Reg);
143 case UniV3S32:
144 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 32) &&
145 MUI.isUniformAtDef(V: Reg);
146 case UniV4S32:
147 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32) &&
148 MUI.isUniformAtDef(V: Reg);
149 case UniV6S32:
150 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 6, ScalarSizeInBits: 32) &&
151 MUI.isUniformAtDef(V: Reg);
152 case UniV8S16:
153 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16) &&
154 MUI.isUniformAtDef(V: Reg);
155 case UniV8S32:
156 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 32) &&
157 MUI.isUniformAtDef(V: Reg);
158 case UniV16S16:
159 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 16) &&
160 MUI.isUniformAtDef(V: Reg);
161 case UniV16S32:
162 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 32) &&
163 MUI.isUniformAtDef(V: Reg);
164 case UniV32S16:
165 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 32, ScalarSizeInBits: 16) &&
166 MUI.isUniformAtDef(V: Reg);
167 case UniV32S32:
168 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 32, ScalarSizeInBits: 32) &&
169 MUI.isUniformAtDef(V: Reg);
170 case UniV2S64:
171 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64) &&
172 MUI.isUniformAtDef(V: Reg);
173 case UniB32:
174 return MRI.getType(Reg).getSizeInBits() == 32 && MUI.isUniformAtDef(V: Reg);
175 case UniB64:
176 return MRI.getType(Reg).getSizeInBits() == 64 && MUI.isUniformAtDef(V: Reg);
177 case UniB96:
178 return MRI.getType(Reg).getSizeInBits() == 96 && MUI.isUniformAtDef(V: Reg);
179 case UniB128:
180 return MRI.getType(Reg).getSizeInBits() == 128 && MUI.isUniformAtDef(V: Reg);
181 case UniB160:
182 return MRI.getType(Reg).getSizeInBits() == 160 && MUI.isUniformAtDef(V: Reg);
183 case UniB256:
184 return MRI.getType(Reg).getSizeInBits() == 256 && MUI.isUniformAtDef(V: Reg);
185 case UniB512:
186 return MRI.getType(Reg).getSizeInBits() == 512 && MUI.isUniformAtDef(V: Reg);
187 case UniBRC: {
188 if (MUI.isDivergentAtDef(V: Reg))
189 return false;
190 // Check if there is SGPR register class of same size as the LLT.
191 const SIRegisterInfo *TRI =
192 static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
193 // There is no 16 bit SGPR register class. Extra size check is required
194 // since getSGPRClassForBitWidth returns SReg_32RegClass for Size 16.
195 unsigned LLTSize = MRI.getType(Reg).getSizeInBits();
196 return LLTSize >= 32 && TRI->getSGPRClassForBitWidth(BitWidth: LLTSize);
197 }
198 case DivS1:
199 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 1) && MUI.isDivergentAtDef(V: Reg);
200 case DivS16:
201 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 16) && MUI.isDivergentAtDef(V: Reg);
202 case DivS32:
203 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 32) && MUI.isDivergentAtDef(V: Reg);
204 case DivS64:
205 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 64) && MUI.isDivergentAtDef(V: Reg);
206 case DivS128:
207 return MRI.getType(Reg) == LLT::scalar(SizeInBits: 128) && MUI.isDivergentAtDef(V: Reg);
208 case DivP0:
209 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 0, SizeInBits: 64) && MUI.isDivergentAtDef(V: Reg);
210 case DivP1:
211 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 1, SizeInBits: 64) && MUI.isDivergentAtDef(V: Reg);
212 case DivP2:
213 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 2, SizeInBits: 32) && MUI.isDivergentAtDef(V: Reg);
214 case DivP3:
215 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 3, SizeInBits: 32) && MUI.isDivergentAtDef(V: Reg);
216 case DivP4:
217 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 4, SizeInBits: 64) && MUI.isDivergentAtDef(V: Reg);
218 case DivP5:
219 return MRI.getType(Reg) == LLT::pointer(AddressSpace: 5, SizeInBits: 32) && MUI.isDivergentAtDef(V: Reg);
220 case DivPtr32:
221 return isAnyPtr(Ty: MRI.getType(Reg), Width: 32) && MUI.isDivergentAtDef(V: Reg);
222 case DivPtr64:
223 return isAnyPtr(Ty: MRI.getType(Reg), Width: 64) && MUI.isDivergentAtDef(V: Reg);
224 case DivPtr128:
225 return isAnyPtr(Ty: MRI.getType(Reg), Width: 128) && MUI.isDivergentAtDef(V: Reg);
226 case DivV2S16:
227 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16) &&
228 MUI.isDivergentAtDef(V: Reg);
229 case DivV2S32:
230 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32) &&
231 MUI.isDivergentAtDef(V: Reg);
232 case DivV4S32:
233 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32) &&
234 MUI.isDivergentAtDef(V: Reg);
235 case DivV2S64:
236 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64) &&
237 MUI.isDivergentAtDef(V: Reg);
238 case DivV3S32:
239 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 32) &&
240 MUI.isDivergentAtDef(V: Reg);
241 case DivV4S16:
242 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16) &&
243 MUI.isDivergentAtDef(V: Reg);
244 case DivV8S16:
245 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16) &&
246 MUI.isDivergentAtDef(V: Reg);
247 case DivV8S32:
248 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 32) &&
249 MUI.isDivergentAtDef(V: Reg);
250 case DivV16S16:
251 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 16) &&
252 MUI.isDivergentAtDef(V: Reg);
253 case DivV16S32:
254 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 32) &&
255 MUI.isDivergentAtDef(V: Reg);
256 case DivV6S32:
257 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 6, ScalarSizeInBits: 32) &&
258 MUI.isDivergentAtDef(V: Reg);
259 case DivV32S16:
260 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 32, ScalarSizeInBits: 16) &&
261 MUI.isDivergentAtDef(V: Reg);
262 case DivV32S32:
263 return MRI.getType(Reg) == LLT::fixed_vector(NumElements: 32, ScalarSizeInBits: 32) &&
264 MUI.isDivergentAtDef(V: Reg);
265 case DivB32:
266 return MRI.getType(Reg).getSizeInBits() == 32 && MUI.isDivergentAtDef(V: Reg);
267 case DivB64:
268 return MRI.getType(Reg).getSizeInBits() == 64 && MUI.isDivergentAtDef(V: Reg);
269 case DivB96:
270 return MRI.getType(Reg).getSizeInBits() == 96 && MUI.isDivergentAtDef(V: Reg);
271 case DivB128:
272 return MRI.getType(Reg).getSizeInBits() == 128 && MUI.isDivergentAtDef(V: Reg);
273 case DivB160:
274 return MRI.getType(Reg).getSizeInBits() == 160 && MUI.isDivergentAtDef(V: Reg);
275 case DivB256:
276 return MRI.getType(Reg).getSizeInBits() == 256 && MUI.isDivergentAtDef(V: Reg);
277 case DivB512:
278 return MRI.getType(Reg).getSizeInBits() == 512 && MUI.isDivergentAtDef(V: Reg);
279 case DivBRC: {
280 if (MUI.isUniformAtDef(V: Reg))
281 return false;
282 // Check if there is VGPR register class of same size as the LLT.
283 const SIRegisterInfo *TRI =
284 static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
285 return TRI->getSGPRClassForBitWidth(BitWidth: MRI.getType(Reg).getSizeInBits());
286 }
287 case BRC: {
288 // Check if there is SGPR and VGPR register class of same size as the LLT.
289 const SIRegisterInfo *TRI =
290 static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
291 unsigned LLTSize = MRI.getType(Reg).getSizeInBits();
292 return LLTSize >= 32 && TRI->getSGPRClassForBitWidth(BitWidth: LLTSize) &&
293 TRI->getVGPRClassForBitWidth(BitWidth: LLTSize);
294 }
295 case _:
296 return true;
297 default:
298 llvm_unreachable("missing matchUniformityAndLLT");
299 }
300}
301
302bool PredicateMapping::match(const MachineInstr &MI,
303 const MachineUniformityInfo &MUI,
304 const MachineRegisterInfo &MRI) const {
305 // Check LLT signature.
306 for (unsigned i = 0; i < OpUniformityAndTypes.size(); ++i) {
307 const MachineOperand &MO = MI.getOperand(i);
308 if (OpUniformityAndTypes[i] == _) {
309 assert((!MI.getOperand(i).isReg() ||
310 !MI.getOperand(i).getReg().isVirtual()) &&
311 "_ is for non-register and physical register operands only");
312 continue;
313 }
314
315 // Remaining IDs check registers.
316 if (!MO.isReg())
317 return false;
318
319 if (!matchUniformityAndLLT(Reg: MO.getReg(), UniID: OpUniformityAndTypes[i], MUI, MRI))
320 return false;
321 }
322
323 // More complex check.
324 if (TestFunc)
325 return TestFunc(MI);
326
327 return true;
328}
329
330SetOfRulesForOpcode::SetOfRulesForOpcode() = default;
331
332SetOfRulesForOpcode::SetOfRulesForOpcode(FastRulesTypes FastTypes)
333 : FastTypes(FastTypes) {}
334
335UniformityLLTOpPredicateID LLTToId(LLT Ty) {
336 if (Ty == LLT::scalar(SizeInBits: 16))
337 return S16;
338 if (Ty == LLT::scalar(SizeInBits: 32))
339 return S32;
340 if (Ty == LLT::scalar(SizeInBits: 64))
341 return S64;
342 if (Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16))
343 return V2S16;
344 if (Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32))
345 return V2S32;
346 if (Ty == LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 32))
347 return V3S32;
348 if (Ty == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32))
349 return V4S32;
350 return _;
351}
352
353UniformityLLTOpPredicateID LLTToBId(LLT Ty) {
354 if (Ty == LLT::scalar(SizeInBits: 32) || Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16) ||
355 isAnyPtr(Ty, Width: 32))
356 return B32;
357 if (Ty == LLT::scalar(SizeInBits: 64) || Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32) ||
358 Ty == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16) || isAnyPtr(Ty, Width: 64))
359 return B64;
360 if (Ty == LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 32))
361 return B96;
362 if (Ty == LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32) || Ty == LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64) ||
363 Ty == LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16) || isAnyPtr(Ty, Width: 128))
364 return B128;
365 return _;
366}
367
368const RegBankLLTMapping *
369SetOfRulesForOpcode::findMappingForMI(const MachineInstr &MI,
370 const MachineRegisterInfo &MRI,
371 const MachineUniformityInfo &MUI) const {
372 // Search in "Fast Rules".
373 // Note: if fast rules are enabled, RegBankLLTMapping must be added in each
374 // slot that could "match fast Predicate". If not, InvalidMapping is
375 // returned which results in failure, does not search "Slow Rules".
376 if (FastTypes != NoFastRules) {
377 Register Reg = MI.getOperand(i: 0).getReg();
378 int Slot;
379 if (FastTypes == StandardB)
380 Slot = getFastPredicateSlot(Ty: LLTToBId(Ty: MRI.getType(Reg)));
381 else
382 Slot = getFastPredicateSlot(Ty: LLTToId(Ty: MRI.getType(Reg)));
383
384 if (Slot != -1)
385 return MUI.isUniformAtDef(V: Reg) ? &Uni[Slot] : &Div[Slot];
386 }
387
388 // Slow search for more complex rules.
389 for (const RegBankLegalizeRule &Rule : Rules) {
390 if (Rule.Predicate.match(MI, MUI, MRI))
391 return &Rule.OperandMapping;
392 }
393
394 return nullptr;
395}
396
397void SetOfRulesForOpcode::addRule(RegBankLegalizeRule Rule) {
398 Rules.push_back(Elt: Rule);
399}
400
401void SetOfRulesForOpcode::addFastRuleDivergent(UniformityLLTOpPredicateID Ty,
402 RegBankLLTMapping RuleApplyIDs) {
403 int Slot = getFastPredicateSlot(Ty);
404 assert(Slot != -1 && "Ty unsupported in this FastRulesTypes");
405 Div[Slot] = std::move(RuleApplyIDs);
406}
407
408void SetOfRulesForOpcode::addFastRuleUniform(UniformityLLTOpPredicateID Ty,
409 RegBankLLTMapping RuleApplyIDs) {
410 int Slot = getFastPredicateSlot(Ty);
411 assert(Slot != -1 && "Ty unsupported in this FastRulesTypes");
412 Uni[Slot] = std::move(RuleApplyIDs);
413}
414
415int SetOfRulesForOpcode::getFastPredicateSlot(
416 UniformityLLTOpPredicateID Ty) const {
417 switch (FastTypes) {
418 case Standard: {
419 switch (Ty) {
420 case S32:
421 return 0;
422 case S16:
423 return 1;
424 case S64:
425 return 2;
426 case V2S16:
427 return 3;
428 default:
429 return -1;
430 }
431 }
432 case StandardB: {
433 switch (Ty) {
434 case B32:
435 return 0;
436 case B64:
437 return 1;
438 case B96:
439 return 2;
440 case B128:
441 return 3;
442 default:
443 return -1;
444 }
445 }
446 case Vector: {
447 switch (Ty) {
448 case S32:
449 return 0;
450 case V2S32:
451 return 1;
452 case V3S32:
453 return 2;
454 case V4S32:
455 return 3;
456 default:
457 return -1;
458 }
459 }
460 default:
461 return -1;
462 }
463}
464
465RegBankLegalizeRules::RuleSetInitializer
466RegBankLegalizeRules::addRulesForGOpcs(std::initializer_list<unsigned> OpcList,
467 FastRulesTypes FastTypes) {
468 return RuleSetInitializer(OpcList, GRulesAlias, GRules, FastTypes);
469}
470
471RegBankLegalizeRules::RuleSetInitializer
472RegBankLegalizeRules::addRulesForIOpcs(std::initializer_list<unsigned> OpcList,
473 FastRulesTypes FastTypes) {
474 return RuleSetInitializer(OpcList, IRulesAlias, IRules, FastTypes);
475}
476
477const SetOfRulesForOpcode *
478RegBankLegalizeRules::getRulesForOpc(MachineInstr &MI) const {
479 unsigned Opc = MI.getOpcode();
480 if (Opc == AMDGPU::G_INTRINSIC || Opc == AMDGPU::G_INTRINSIC_CONVERGENT ||
481 Opc == AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS ||
482 Opc == AMDGPU::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS) {
483 unsigned IntrID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
484 auto IRAIt = IRulesAlias.find(Val: IntrID);
485 if (IRAIt == IRulesAlias.end())
486 return nullptr;
487 return &IRules.at(Val: IRAIt->second);
488 }
489
490 auto GRAIt = GRulesAlias.find(Val: Opc);
491 if (GRAIt == GRulesAlias.end())
492 return nullptr;
493 return &GRules.at(Val: GRAIt->second);
494}
495
496// Syntactic sugar wrapper for predicate lambda that enables '&&', '||' and '!'.
497class Predicate {
498private:
499 struct Elt {
500 // Save formula composed of Pred, '&&', '||' and '!' as a jump table.
501 // Sink ! to Pred. For example !((A && !B) || C) -> (!A || B) && !C
502 // Sequences of && and || will be represented by jumps, for example:
503 // (A && B && ... X) or (A && B && ... X) || Y
504 // A == true jump to B
505 // A == false jump to end or Y, result is A(false) or Y
506 // (A || B || ... X) or (A || B || ... X) && Y
507 // A == true jump to end or Y, result is A(true) or Y
508 // A == false jump to B
509 // Notice that when negating expression, we simply flip Neg on each Pred
510 // and swap TJumpOffset and FJumpOffset (&& becomes ||, || becomes &&).
511 std::function<bool(const MachineInstr &)> Pred;
512 bool Neg; // Neg of Pred is calculated before jump
513 unsigned TJumpOffset;
514 unsigned FJumpOffset;
515 };
516
517 SmallVector<Elt, 8> Expression;
518
519 Predicate(SmallVectorImpl<Elt> &&Expr) { Expression.swap(RHS&: Expr); };
520
521public:
522 Predicate(std::function<bool(const MachineInstr &)> Pred) {
523 Expression.push_back(Elt: {.Pred: Pred, .Neg: false, .TJumpOffset: 1, .FJumpOffset: 1});
524 };
525
526 bool operator()(const MachineInstr &MI) const {
527 unsigned Idx = 0;
528 unsigned ResultIdx = Expression.size();
529 bool Result;
530 do {
531 Result = Expression[Idx].Pred(MI);
532 Result = Expression[Idx].Neg ? !Result : Result;
533 if (Result) {
534 Idx += Expression[Idx].TJumpOffset;
535 } else {
536 Idx += Expression[Idx].FJumpOffset;
537 }
538 } while ((Idx != ResultIdx));
539
540 return Result;
541 };
542
543 Predicate operator!() const {
544 SmallVector<Elt, 8> NegExpression;
545 for (const Elt &ExprElt : Expression) {
546 NegExpression.push_back(Elt: {.Pred: ExprElt.Pred, .Neg: !ExprElt.Neg, .TJumpOffset: ExprElt.FJumpOffset,
547 .FJumpOffset: ExprElt.TJumpOffset});
548 }
549 return Predicate(std::move(NegExpression));
550 };
551
552 Predicate operator&&(const Predicate &RHS) const {
553 SmallVector<Elt, 8> AndExpression = Expression;
554
555 unsigned RHSSize = RHS.Expression.size();
556 unsigned ResultIdx = Expression.size();
557 for (unsigned i = 0; i < ResultIdx; ++i) {
558 // LHS results in false, whole expression results in false.
559 if (i + AndExpression[i].FJumpOffset == ResultIdx)
560 AndExpression[i].FJumpOffset += RHSSize;
561 }
562
563 AndExpression.append(RHS: RHS.Expression);
564
565 return Predicate(std::move(AndExpression));
566 }
567
568 Predicate operator||(const Predicate &RHS) const {
569 SmallVector<Elt, 8> OrExpression = Expression;
570
571 unsigned RHSSize = RHS.Expression.size();
572 unsigned ResultIdx = Expression.size();
573 for (unsigned i = 0; i < ResultIdx; ++i) {
574 // LHS results in true, whole expression results in true.
575 if (i + OrExpression[i].TJumpOffset == ResultIdx)
576 OrExpression[i].TJumpOffset += RHSSize;
577 }
578
579 OrExpression.append(RHS: RHS.Expression);
580
581 return Predicate(std::move(OrExpression));
582 }
583};
584
585// Initialize rules
586RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
587 MachineRegisterInfo &_MRI)
588 : ST(&_ST), MRI(&_MRI) {
589
590 addRulesForGOpcs(OpcList: {G_ADD, G_SUB}, FastTypes: Standard)
591 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr32Trunc}, {Sgpr32AExt, Sgpr32AExt}})
592 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
593 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}})
594 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
595 .Uni(Ty: V2S16, RuleApplyIDs: {{SgprV2S16}, {SgprV2S16, SgprV2S16}, UnpackAExt})
596 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
597 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {Sgpr64, Sgpr64}})
598 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr64}})
599 .Any(Init: {.Predicate: {UniV2S64}, .OperandMapping: {{UniInVgprV2S64}, {VgprV2S64, VgprV2S64}}})
600 .Any(Init: {.Predicate: {DivV2S64}, .OperandMapping: {{VgprV2S64}, {VgprV2S64, VgprV2S64}}});
601
602 addRulesForGOpcs(OpcList: {G_UADDO, G_USUBO}, FastTypes: Standard)
603 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32, Sgpr32Trunc}, {Sgpr32, Sgpr32}})
604 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32, Vcc}, {Vgpr32, Vgpr32}});
605
606 addRulesForGOpcs(OpcList: {G_UADDE, G_USUBE, G_SADDE, G_SSUBE}, FastTypes: Standard)
607 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32, Sgpr32Trunc}, {Sgpr32, Sgpr32, Sgpr32AExtBoolInReg}})
608 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32, Vcc}, {Vgpr32, Vgpr32, Vcc}});
609
610 addRulesForGOpcs(OpcList: {G_UADDSAT, G_SADDSAT, G_USUBSAT, G_SSUBSAT}, FastTypes: Standard)
611 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16, Vgpr16}})
612 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
613 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32}})
614 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
615 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16}})
616 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}});
617
618 bool UseVecMulU64 = ST->useVMulU64Inst();
619 addRulesForGOpcs(OpcList: {G_MUL}, FastTypes: Standard)
620 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
621 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}})
622 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
623 .Uni(Ty: S64, RuleApplyIDs: {{SgprB64}, {SgprB64, SgprB64}})
624 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16}})
625 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
626 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr32Trunc}, {Sgpr32AExt, Sgpr32AExt}})
627 .Div(Ty: S64, RuleApplyIDs: {{VgprB64}, {VgprB64, VgprB64}}, STPred: UseVecMulU64)
628 .Div(Ty: S64, RuleApplyIDs: {{VgprB64}, {VgprB64, VgprB64}, SplitTo32Mul}, STPred: !UseVecMulU64);
629
630 bool hasMulHi = ST->hasScalarMulHiInsts();
631 addRulesForGOpcs(OpcList: {G_UMULH, G_SMULH}, FastTypes: Standard)
632 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
633 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}}, STPred: hasMulHi)
634 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32}}, STPred: !hasMulHi);
635
636 addRulesForGOpcs(OpcList: {G_AMDGPU_MAD_U64_U32}, FastTypes: Standard)
637 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64, Vcc}, {Vgpr32, Vgpr32, Vgpr64}})
638 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64, Sgpr32Trunc}, {Sgpr32, Sgpr32, Sgpr64}, UniMAD64});
639
640 bool HasScalarSMulU64 = ST->hasScalarSMulU64();
641 addRulesForGOpcs(OpcList: {G_AMDGPU_S_MUL_U64_U32, G_AMDGPU_S_MUL_I64_I32}, FastTypes: Standard)
642 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {Sgpr64, Sgpr64}, UniMul64}, STPred: HasScalarSMulU64)
643 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr64}, DivSMulToMAD});
644
645 addRulesForGOpcs(OpcList: {G_XOR, G_OR, G_AND}, FastTypes: StandardB)
646 .Any(Init: {.Predicate: {UniS1}, .OperandMapping: {{Sgpr32Trunc}, {Sgpr32AExt, Sgpr32AExt}}})
647 .Any(Init: {.Predicate: {DivS1}, .OperandMapping: {{Vcc}, {Vcc, Vcc}}})
648 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{Sgpr16}, {Sgpr16, Sgpr16}}})
649 .Any(Init: {.Predicate: {DivS16}, .OperandMapping: {{Vgpr16}, {Vgpr16, Vgpr16}}})
650 .Uni(Ty: B32, RuleApplyIDs: {{SgprB32}, {SgprB32, SgprB32}})
651 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {VgprB32, VgprB32}})
652 .Uni(Ty: B64, RuleApplyIDs: {{SgprB64}, {SgprB64, SgprB64}})
653 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {VgprB64, VgprB64}, SplitTo32});
654
655 addRulesForGOpcs(OpcList: {G_SHL}, FastTypes: Standard)
656 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr32Trunc}, {Sgpr32AExt, Sgpr32ZExt}})
657 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
658 .Uni(Ty: V2S16, RuleApplyIDs: {{SgprV2S16}, {SgprV2S16, SgprV2S16}, UnpackBitShift})
659 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
660 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}})
661 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {Sgpr64, Sgpr32}})
662 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
663 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr32}});
664
665 addRulesForGOpcs(OpcList: {G_LSHR}, FastTypes: Standard)
666 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr32Trunc}, {Sgpr32ZExt, Sgpr32ZExt}})
667 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
668 .Uni(Ty: V2S16, RuleApplyIDs: {{SgprV2S16}, {SgprV2S16, SgprV2S16}, UnpackBitShift})
669 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
670 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}})
671 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {Sgpr64, Sgpr32}})
672 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
673 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr32}});
674
675 addRulesForGOpcs(OpcList: {G_ASHR}, FastTypes: Standard)
676 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr32Trunc}, {Sgpr32SExt, Sgpr32ZExt}})
677 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
678 .Uni(Ty: V2S16, RuleApplyIDs: {{SgprV2S16}, {SgprV2S16, SgprV2S16}, UnpackBitShift})
679 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
680 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}})
681 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {Sgpr64, Sgpr32}})
682 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
683 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr32}});
684
685 addRulesForGOpcs(OpcList: {G_FSHR}, FastTypes: Standard)
686 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32, Vgpr32}})
687 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32, Vgpr32}});
688
689 addRulesForGOpcs(OpcList: {G_BSWAP}, FastTypes: Standard)
690 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16}})
691 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16}})
692 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32}})
693 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32}})
694 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {VgprV2S16}})
695 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16}});
696
697 addRulesForGOpcs(OpcList: {G_AMDGPU_CVT_F32_UBYTE0, G_AMDGPU_CVT_F32_UBYTE1,
698 G_AMDGPU_CVT_F32_UBYTE2, G_AMDGPU_CVT_F32_UBYTE3,
699 G_AMDGPU_RCP_IFLAG},
700 FastTypes: Standard)
701 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32}})
702 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32}});
703
704 addRulesForGOpcs(OpcList: {G_FRAME_INDEX}).Any(Init: {.Predicate: {UniP5, _}, .OperandMapping: {{SgprP5}, {None}}});
705
706 addRulesForGOpcs(OpcList: {G_UBFX, G_SBFX}, FastTypes: Standard)
707 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32, Sgpr32}, S_BFE})
708 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32, Vgpr32}})
709 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {Sgpr64, Sgpr32, Sgpr32}, S_BFE})
710 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr32, Vgpr32}, V_BFE});
711
712 addRulesForGOpcs(OpcList: {G_SMIN, G_SMAX}, FastTypes: Standard)
713 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr32Trunc}, {Sgpr32SExt, Sgpr32SExt}})
714 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
715 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}})
716 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
717 .Uni(Ty: V2S16, RuleApplyIDs: {{SgprV2S16}, {SgprV2S16, SgprV2S16}, UnpackMinMax})
718 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
719 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64, Vgpr64}})
720 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr64}});
721
722 addRulesForGOpcs(OpcList: {G_UMIN, G_UMAX}, FastTypes: Standard)
723 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr32Trunc}, {Sgpr32ZExt, Sgpr32ZExt}})
724 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
725 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}})
726 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
727 .Uni(Ty: V2S16, RuleApplyIDs: {{SgprV2S16}, {SgprV2S16, SgprV2S16}, UnpackMinMax})
728 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
729 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64, Vgpr64}})
730 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr64}});
731
732 addRulesForGOpcs(OpcList: {G_IMPLICIT_DEF})
733 .Any(Init: {.Predicate: {UniS1}, .OperandMapping: {{Sgpr32Trunc}, {}}})
734 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{Sgpr16}, {}}})
735 .Any(Init: {.Predicate: {UniBRC}, .OperandMapping: {{SgprBRC}, {}}});
736
737 addRulesForGOpcs(OpcList: {G_CONSTANT}, FastTypes: Standard)
738 .Any(Init: {.Predicate: {UniS1, _}, .OperandMapping: {{Sgpr32Trunc}, {}, UniCstExt}})
739 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {}})
740 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {}})
741 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {}})
742 .Any(Init: {.Predicate: {UniPtr32, _}, .OperandMapping: {{SgprPtr32}, {}}})
743 .Any(Init: {.Predicate: {UniPtr64, _}, .OperandMapping: {{SgprPtr64}, {}}});
744
745 addRulesForGOpcs(OpcList: {G_FCONSTANT}, FastTypes: Standard)
746 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {}})
747 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {}})
748 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {}});
749
750 addRulesForGOpcs(OpcList: {G_FREEZE})
751 .Any(Init: {.Predicate: {UniS1}, .OperandMapping: {{Sgpr32Trunc}, {Sgpr32AExt}}})
752 .Any(Init: {.Predicate: {DivS1}, .OperandMapping: {{Vcc}, {Vcc}}})
753 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{Sgpr16}, {Sgpr16}}})
754 .Any(Init: {.Predicate: {UniBRC}, .OperandMapping: {{SgprBRC}, {SgprBRC}}})
755 .Any(Init: {.Predicate: {DivBRC}, .OperandMapping: {{VgprBRC}, {VgprBRC}}});
756
757 addRulesForGOpcs(OpcList: {G_BITCAST})
758 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{Sgpr16}, {Sgpr16}}})
759 .Any(Init: {.Predicate: {DivS16}, .OperandMapping: {{Vgpr16}, {Vgpr16}}})
760 .Any(Init: {.Predicate: {UniBRC}, .OperandMapping: {{SgprBRC}, {SgprBRC}}})
761 .Any(Init: {.Predicate: {DivBRC}, .OperandMapping: {{VgprBRC}, {VgprBRC}}});
762
763 addRulesForGOpcs(OpcList: {G_UNMERGE_VALUES})
764 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{}, {}, UnmergeToShiftTrunc}})
765 .Any(Init: {.Predicate: {UniBRC}, .OperandMapping: {{}, {}, VerifyAllSgpr}})
766 .Any(Init: {.Predicate: {DivBRC}, .OperandMapping: {{}, {}, ApplyAllVgpr}});
767
768 addRulesForGOpcs(OpcList: {G_BUILD_VECTOR, G_MERGE_VALUES})
769 .Any(Init: {.Predicate: {UniBRC, S16}, .OperandMapping: {{}, {}, VerifyAllSgpr}})
770 .Any(Init: {.Predicate: {UniBRC, BRC}, .OperandMapping: {{}, {}, VerifyAllSgpr}})
771 .Any(Init: {.Predicate: {DivBRC, S16}, .OperandMapping: {{}, {}, ApplyAllVgpr}})
772 .Any(Init: {.Predicate: {DivBRC, BRC}, .OperandMapping: {{}, {}, ApplyAllVgpr}});
773
774 addRulesForGOpcs(OpcList: {G_CONCAT_VECTORS})
775 .Any(Init: {.Predicate: {UniBRC, BRC}, .OperandMapping: {{}, {}, VerifyAllSgpr}})
776 .Any(Init: {.Predicate: {DivBRC, BRC}, .OperandMapping: {{}, {}, ApplyAllVgpr}});
777
778 addRulesForGOpcs(OpcList: {G_PHI})
779 .Any(Init: {.Predicate: {UniS1}, .OperandMapping: {{}, {}, AextToS32InIncomingBlockGPHI}})
780 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{}, {}, VerifyAllSgprGPHI}})
781 .Any(Init: {.Predicate: {UniBRC}, .OperandMapping: {{}, {}, VerifyAllSgprGPHI}})
782 .Any(Init: {.Predicate: {DivBRC}, .OperandMapping: {{}, {}, VerifyAllSgprOrVgprGPHI}});
783
784 addRulesForGOpcs(OpcList: {G_EXTRACT_VECTOR_ELT})
785 .Any(Init: {.Predicate: {UniB32, UniBRC, UniS32}, .OperandMapping: {{SgprB32}, {SgprBRC, Sgpr32}}})
786 .Any(Init: {.Predicate: {DivB32, DivBRC, UniS32}, .OperandMapping: {{VgprB32}, {VgprBRC, Sgpr32}}})
787 .Any(Init: {.Predicate: {DivB32, BRC, DivS32},
788 .OperandMapping: {{VgprB32}, {VgprBRC, Vgpr32}, ExtrVecEltToSel}})
789 .Any(Init: {.Predicate: {UniB64, UniBRC, UniS32}, .OperandMapping: {{SgprB64}, {SgprBRC, Sgpr32}}})
790 .Any(Init: {.Predicate: {DivB64, DivBRC, UniS32},
791 .OperandMapping: {{VgprB64}, {VgprBRC, Sgpr32}, ExtrVecEltTo32}})
792 .Any(Init: {.Predicate: {DivB64, BRC, DivS32},
793 .OperandMapping: {{VgprB64}, {VgprBRC, Vgpr32}, ExtrVecEltToSel}});
794
795 addRulesForGOpcs(OpcList: {G_INSERT_VECTOR_ELT})
796 .Any(Init: {.Predicate: {UniBRC, UniBRC, UniB32, UniS32},
797 .OperandMapping: {{SgprBRC}, {SgprBRC, SgprB32, Sgpr32}}})
798 .Any(
799 Init: {.Predicate: {DivBRC, BRC, B32, UniS32}, .OperandMapping: {{VgprBRC}, {VgprBRC, VgprB32, Sgpr32}}})
800 .Any(Init: {.Predicate: {DivBRC, BRC, B32, DivS32},
801 .OperandMapping: {{VgprBRC}, {VgprBRC, VgprB32, Vgpr32}, InsVecEltToSel}})
802 .Any(Init: {.Predicate: {UniBRC, UniBRC, UniB64, UniS32},
803 .OperandMapping: {{SgprBRC}, {SgprBRC, SgprB64, Sgpr32}, InsVecEltToSel}})
804 .Any(Init: {.Predicate: {DivBRC, BRC, B64, UniS32},
805 .OperandMapping: {{VgprBRC}, {VgprBRC, VgprB64, Sgpr32}, InsVecEltTo32}})
806 .Any(Init: {.Predicate: {DivBRC, BRC, B64, DivS32},
807 .OperandMapping: {{VgprBRC}, {VgprBRC, VgprB64, Vgpr32}, InsVecEltToSel}});
808
809 // INTERSECT_RAY {Div}, {{VgprDst...}, {VgprSrc, ..., Sgpr_WF_RsrcIdx}}
810 // INTERSECT_RAY {Uni}, {{UniInVgprDst...}, {VgprSrc, ..., Sgpr_WF_RsrcIdx}}
811 addRulesForGOpcs(OpcList: {G_AMDGPU_BVH_INTERSECT_RAY, G_AMDGPU_BVH_DUAL_INTERSECT_RAY,
812 G_AMDGPU_BVH8_INTERSECT_RAY})
813 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {}, ApplyBVH_INTERSECT_RAY}});
814
815 // LOAD {Div}, {{VgprDst...}, {VgprSrc, ..., Sgpr_WF_RsrcIdx}}
816 // LOAD {Uni}, {{UniInVgprDst...}, {VgprSrc, ..., Sgpr_WF_RsrcIdx}}
817 // LOAD_NORET {}, {{}, {Imm, VgprSrc, ..., Sgpr_WF_RsrcIdx}}
818 // STORE {}, {{}, {VgprSrc, ..., Sgpr_WF_RsrcIdx}}
819 addRulesForGOpcs(OpcList: {G_AMDGPU_INTRIN_IMAGE_LOAD, G_AMDGPU_INTRIN_IMAGE_LOAD_D16,
820 G_AMDGPU_INTRIN_IMAGE_LOAD_NORET,
821 G_AMDGPU_INTRIN_IMAGE_STORE,
822 G_AMDGPU_INTRIN_IMAGE_STORE_D16})
823 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {}, ApplyINTRIN_IMAGE}});
824
825 Predicate isSignedICmp([](const MachineInstr &MI) -> bool {
826 auto Pred =
827 static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
828 return CmpInst::isSigned(Pred);
829 });
830
831 Predicate isEqualityICmp([](const MachineInstr &MI) -> bool {
832 auto Pred =
833 static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
834 return ICmpInst::isEquality(P: Pred);
835 });
836
837 bool HasScalarCompareEq64 = ST->hasScalarCompareEq64();
838 // clang-format off
839 addRulesForGOpcs(OpcList: {G_ICMP})
840 .Any(Init: {.Predicate: {{UniS1, _, S16}, isEqualityICmp}, .OperandMapping: {{Sgpr32Trunc}, {None, Sgpr32ZExt, Sgpr32ZExt}}})
841 .Any(Init: {.Predicate: {{UniS1, _, S16}, !isEqualityICmp && isSignedICmp}, .OperandMapping: {{Sgpr32Trunc}, {None, Sgpr32SExt, Sgpr32SExt}}})
842 .Any(Init: {.Predicate: {{UniS1, _, S16}, !isEqualityICmp && !isSignedICmp}, .OperandMapping: {{Sgpr32Trunc}, {None, Sgpr32ZExt, Sgpr32ZExt}}})
843 .Any(Init: {.Predicate: {{DivS1, _, S16}}, .OperandMapping: {{Vcc}, {None, Vgpr16, Vgpr16}}})
844 .Any(Init: {.Predicate: {{UniS1, _, S32}}, .OperandMapping: {{Sgpr32Trunc}, {None, Sgpr32, Sgpr32}}})
845 .Any(Init: {.Predicate: {{DivS1, _, S32}}, .OperandMapping: {{Vcc}, {None, Vgpr32, Vgpr32}}})
846 .Any(Init: {.Predicate: {{UniS1, _, S64}, isEqualityICmp}, .OperandMapping: {{Sgpr32Trunc}, {None, Sgpr64, Sgpr64}}}, STPred: HasScalarCompareEq64)
847 .Any(Init: {.Predicate: {{UniS1, _, S64}, isEqualityICmp}, .OperandMapping: {{UniInVcc}, {None, Vgpr64, Vgpr64}}}, STPred: !HasScalarCompareEq64)
848 .Any(Init: {.Predicate: {{UniS1, _, S64}, !isEqualityICmp}, .OperandMapping: {{UniInVcc}, {None, Vgpr64, Vgpr64}}})
849 .Any(Init: {.Predicate: {{DivS1, _, S64}}, .OperandMapping: {{Vcc}, {None, Vgpr64, Vgpr64}}})
850 .Any(Init: {.Predicate: {{UniS1, _, Ptr32}}, .OperandMapping: {{Sgpr32Trunc}, {None, SgprPtr32, SgprPtr32}}})
851 .Any(Init: {.Predicate: {{DivS1, _, Ptr32}}, .OperandMapping: {{Vcc}, {None, VgprPtr32, VgprPtr32}}})
852 .Any(Init: {.Predicate: {{UniS1, _, Ptr64}, isEqualityICmp}, .OperandMapping: {{Sgpr32Trunc}, {None, SgprPtr64, SgprPtr64}}}, STPred: HasScalarCompareEq64)
853 .Any(Init: {.Predicate: {{UniS1, _, Ptr64}, isEqualityICmp}, .OperandMapping: {{UniInVcc}, {None, VgprPtr64, VgprPtr64}}}, STPred: !HasScalarCompareEq64)
854 .Any(Init: {.Predicate: {{UniS1, _, Ptr64}, !isEqualityICmp}, .OperandMapping: {{UniInVcc}, {None, VgprPtr64, VgprPtr64}}})
855 .Any(Init: {.Predicate: {{DivS1, _, Ptr64}}, .OperandMapping: {{Vcc}, {None, VgprPtr64, VgprPtr64}}});
856 // clang-format on
857
858 addRulesForGOpcs(OpcList: {G_BRCOND})
859 .Any(Init: {.Predicate: {UniS1}, .OperandMapping: {{}, {Sgpr32AExtBoolInReg}}})
860 .Any(Init: {.Predicate: {DivS1}, .OperandMapping: {{}, {Vcc}}});
861
862 addRulesForGOpcs(OpcList: {G_BR}).Any(Init: {.Predicate: {_}, .OperandMapping: {{}, {None}}});
863
864 addRulesForGOpcs(OpcList: {G_SELECT}, FastTypes: StandardB)
865 .Any(Init: {.Predicate: {DivS16}, .OperandMapping: {{Vgpr16}, {Vcc, Vgpr16, Vgpr16}}})
866 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{Sgpr16}, {Sgpr32AExtBoolInReg, Sgpr16, Sgpr16}}})
867 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {Vcc, VgprB32, VgprB32}})
868 .Uni(Ty: B32, RuleApplyIDs: {{SgprB32}, {Sgpr32AExtBoolInReg, SgprB32, SgprB32}})
869 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {Vcc, VgprB64, VgprB64}, SplitTo32Select})
870 .Uni(Ty: B64, RuleApplyIDs: {{SgprB64}, {Sgpr32AExtBoolInReg, SgprB64, SgprB64}});
871
872 addRulesForGOpcs(OpcList: {G_ANYEXT})
873 .Any(Init: {.Predicate: {UniS16, S1}, .OperandMapping: {{None}, {None}}}) // should be combined away
874 .Any(Init: {.Predicate: {UniS32, S1}, .OperandMapping: {{None}, {None}}}) // should be combined away
875 .Any(Init: {.Predicate: {UniS64, S1}, .OperandMapping: {{None}, {None}}}) // should be combined away
876 .Any(Init: {.Predicate: {DivS16, S1}, .OperandMapping: {{Vgpr16}, {Vcc}, VccExtToSel}})
877 .Any(Init: {.Predicate: {DivS32, S1}, .OperandMapping: {{Vgpr32}, {Vcc}, VccExtToSel}})
878 .Any(Init: {.Predicate: {DivS64, S1}, .OperandMapping: {{Vgpr64}, {Vcc}, VccExtToSel}})
879 .Any(Init: {.Predicate: {UniS64, S32}, .OperandMapping: {{Sgpr64}, {Sgpr32}, Ext32To64}})
880 .Any(Init: {.Predicate: {DivS64, S32}, .OperandMapping: {{Vgpr64}, {Vgpr32}, Ext32To64}})
881 .Any(Init: {.Predicate: {UniS64, S16}, .OperandMapping: {{Sgpr64}, {Sgpr32AExt}, Ext32To64}})
882 .Any(Init: {.Predicate: {DivS64, S16}, .OperandMapping: {{Vgpr64}, {Vgpr32AExt}, Ext32To64}})
883 .Any(Init: {.Predicate: {UniS32, S16}, .OperandMapping: {{Sgpr32}, {Sgpr16}}})
884 .Any(Init: {.Predicate: {DivS32, S16}, .OperandMapping: {{Vgpr32}, {Vgpr16}}});
885
886 bool Has16bitCmp = ST->has16BitInsts();
887
888 // In global-isel G_TRUNC in-reg is treated as no-op, inst selected into COPY.
889 // It is up to user to deal with truncated bits.
890 // S1, S16, S32 and S64 results are handled with specific rules. Remaining
891 // (result, source) pairs with valid register classes are covered by the
892 // generic UniBRC/DivBRC wildcard rules.
893 addRulesForGOpcs(OpcList: {G_TRUNC})
894 .Any(Init: {.Predicate: {UniS1, UniS16}, .OperandMapping: {{None}, {None}}}) // should be combined away
895 .Any(Init: {.Predicate: {UniS1, UniS32}, .OperandMapping: {{None}, {None}}}) // should be combined away
896 .Any(Init: {.Predicate: {UniS1, UniS64}, .OperandMapping: {{None}, {None}}}) // should be combined away
897 .Any(Init: {.Predicate: {UniS16, S32}, .OperandMapping: {{Sgpr16}, {Sgpr32}}})
898 .Any(Init: {.Predicate: {UniBRC, UniBRC}, .OperandMapping: {{SgprBRC}, {SgprBRC}}})
899 .Any(Init: {.Predicate: {DivBRC, DivBRC}, .OperandMapping: {{VgprBRC}, {VgprBRC}}})
900 .Any(Init: {.Predicate: {UniV2S16, V2S32}, .OperandMapping: {{SgprV2S16}, {SgprV2S32}}})
901 .Any(Init: {.Predicate: {DivV2S16, V2S32}, .OperandMapping: {{VgprV2S16}, {VgprV2S32}}})
902 // This is non-trivial. VgprToVccCopy is done using compare instruction.
903 .Any(Init: {.Predicate: {DivS1, DivS16}, .OperandMapping: {{Vcc}, {Vgpr16}, VgprToVccCopy}}, STPred: Has16bitCmp)
904 .Any(Init: {.Predicate: {DivS1, DivS16}, .OperandMapping: {{Vcc}, {Vgpr32AExt}, VgprToVccCopy}},
905 STPred: !Has16bitCmp)
906 .Any(Init: {.Predicate: {DivS1, DivS32}, .OperandMapping: {{Vcc}, {Vgpr32}, VgprToVccCopy}})
907 .Any(Init: {.Predicate: {DivS1, DivS64}, .OperandMapping: {{Vcc}, {Vgpr64}, VgprToVccCopy}});
908
909 addRulesForGOpcs(OpcList: {G_ZEXT})
910 .Any(Init: {.Predicate: {UniS16, S1}, .OperandMapping: {{Sgpr32Trunc}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
911 .Any(Init: {.Predicate: {UniS32, S1}, .OperandMapping: {{Sgpr32}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
912 .Any(Init: {.Predicate: {UniS64, S1}, .OperandMapping: {{Sgpr64}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
913 .Any(Init: {.Predicate: {DivS16, S1}, .OperandMapping: {{Vgpr16}, {Vcc}, VccExtToSel}})
914 .Any(Init: {.Predicate: {DivS32, S1}, .OperandMapping: {{Vgpr32}, {Vcc}, VccExtToSel}})
915 .Any(Init: {.Predicate: {DivS64, S1}, .OperandMapping: {{Vgpr64}, {Vcc}, VccExtToSel}})
916 .Any(Init: {.Predicate: {UniS64, S32}, .OperandMapping: {{Sgpr64}, {Sgpr32}, Ext32To64}})
917 .Any(Init: {.Predicate: {DivS64, S32}, .OperandMapping: {{Vgpr64}, {Vgpr32}, Ext32To64}})
918 // not extending S16 to S32 is questionable.
919 .Any(Init: {.Predicate: {UniS64, S16}, .OperandMapping: {{Sgpr64}, {Sgpr32ZExt}, Ext32To64}})
920 .Any(Init: {.Predicate: {DivS64, S16}, .OperandMapping: {{Vgpr64}, {Vgpr32ZExt}, Ext32To64}})
921 .Any(Init: {.Predicate: {UniS32, S16}, .OperandMapping: {{Sgpr32}, {Sgpr16}}})
922 .Any(Init: {.Predicate: {DivS32, S16}, .OperandMapping: {{Vgpr32}, {Vgpr16}}});
923
924 addRulesForGOpcs(OpcList: {G_SEXT})
925 .Any(Init: {.Predicate: {UniS16, S1}, .OperandMapping: {{Sgpr32Trunc}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
926 .Any(Init: {.Predicate: {UniS32, S1}, .OperandMapping: {{Sgpr32}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
927 .Any(Init: {.Predicate: {UniS64, S1}, .OperandMapping: {{Sgpr64}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
928 .Any(Init: {.Predicate: {DivS16, S1}, .OperandMapping: {{Vgpr16}, {Vcc}, VccExtToSel}})
929 .Any(Init: {.Predicate: {DivS32, S1}, .OperandMapping: {{Vgpr32}, {Vcc}, VccExtToSel}})
930 .Any(Init: {.Predicate: {DivS64, S1}, .OperandMapping: {{Vgpr64}, {Vcc}, VccExtToSel}})
931 .Any(Init: {.Predicate: {UniS64, S32}, .OperandMapping: {{Sgpr64}, {Sgpr32}, Ext32To64}})
932 .Any(Init: {.Predicate: {DivS64, S32}, .OperandMapping: {{Vgpr64}, {Vgpr32}, Ext32To64}})
933 // not extending S16 to S32 is questionable.
934 .Any(Init: {.Predicate: {UniS64, S16}, .OperandMapping: {{Sgpr64}, {Sgpr32SExt}, Ext32To64}})
935 .Any(Init: {.Predicate: {DivS64, S16}, .OperandMapping: {{Vgpr64}, {Vgpr32SExt}, Ext32To64}})
936 .Any(Init: {.Predicate: {UniS32, S16}, .OperandMapping: {{Sgpr32}, {Sgpr16}}})
937 .Any(Init: {.Predicate: {DivS32, S16}, .OperandMapping: {{Vgpr32}, {Vgpr16}}});
938
939 addRulesForGOpcs(OpcList: {G_SEXT_INREG})
940 .Any(Init: {.Predicate: {UniS32, S32}, .OperandMapping: {{Sgpr32}, {Sgpr32}}})
941 .Any(Init: {.Predicate: {DivS32, S32}, .OperandMapping: {{Vgpr32}, {Vgpr32}}})
942 .Any(Init: {.Predicate: {UniS64, S64}, .OperandMapping: {{Sgpr64}, {Sgpr64}}})
943 .Any(Init: {.Predicate: {DivS64, S64}, .OperandMapping: {{Vgpr64}, {Vgpr64}, SplitTo32SExtInReg}});
944
945 addRulesForGOpcs(OpcList: {G_ASSERT_ZEXT, G_ASSERT_SEXT}, FastTypes: Standard)
946 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Imm}})
947 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Imm}})
948 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {Sgpr64, Imm}})
949 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Imm}});
950
951 addRulesForGOpcs(OpcList: {G_ASSERT_ALIGN}, FastTypes: Standard)
952 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32}})
953 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32}})
954 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {Sgpr64}})
955 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64}})
956 .Any(Init: {.Predicate: {UniPtr32}, .OperandMapping: {{SgprPtr32}, {SgprPtr32}}})
957 .Any(Init: {.Predicate: {DivPtr32}, .OperandMapping: {{VgprPtr32}, {VgprPtr32}}})
958 .Any(Init: {.Predicate: {UniPtr64}, .OperandMapping: {{SgprPtr64}, {SgprPtr64}}})
959 .Any(Init: {.Predicate: {DivPtr64}, .OperandMapping: {{VgprPtr64}, {VgprPtr64}}});
960
961 // Atomic read-modify-write operations: result and value are always VGPR,
962 // pointer varies by address space.
963 addRulesForGOpcs(OpcList: {G_ATOMICRMW_ADD, G_ATOMICRMW_SUB, G_ATOMICRMW_XCHG,
964 G_ATOMICRMW_AND, G_ATOMICRMW_OR, G_ATOMICRMW_XOR,
965 G_ATOMICRMW_MIN, G_ATOMICRMW_MAX, G_ATOMICRMW_UMIN,
966 G_ATOMICRMW_UMAX, G_ATOMICRMW_UINC_WRAP,
967 G_ATOMICRMW_UDEC_WRAP, G_ATOMICRMW_FMIN, G_ATOMICRMW_FMAX})
968 .Any(Init: {.Predicate: {DivS32, P0, S32}, .OperandMapping: {{Vgpr32}, {VgprP0, Vgpr32}}})
969 .Any(Init: {.Predicate: {DivS64, P0, S64}, .OperandMapping: {{Vgpr64}, {VgprP0, Vgpr64}}})
970 .Any(Init: {.Predicate: {DivS32, P1, S32}, .OperandMapping: {{Vgpr32}, {VgprP1, Vgpr32}}})
971 .Any(Init: {.Predicate: {DivS64, P1, S64}, .OperandMapping: {{Vgpr64}, {VgprP1, Vgpr64}}})
972 .Any(Init: {.Predicate: {DivS32, P3, S32}, .OperandMapping: {{Vgpr32}, {VgprP3, Vgpr32}}})
973 .Any(Init: {.Predicate: {DivS64, P3, S64}, .OperandMapping: {{Vgpr64}, {VgprP3, Vgpr64}}});
974
975 addRulesForGOpcs(OpcList: {G_ATOMICRMW_USUB_SAT, G_ATOMICRMW_USUB_COND})
976 .Any(Init: {.Predicate: {DivS32, P0}, .OperandMapping: {{Vgpr32}, {VgprP0, Vgpr32}}})
977 .Any(Init: {.Predicate: {DivS32, P1}, .OperandMapping: {{Vgpr32}, {VgprP1, Vgpr32}}})
978 .Any(Init: {.Predicate: {DivS32, P3}, .OperandMapping: {{Vgpr32}, {VgprP3, Vgpr32}}});
979
980 bool HasAtomicFlatPkAdd16Insts = ST->hasAtomicFlatPkAdd16Insts();
981 bool HasAtomicBufferGlobalPkAddF16Insts =
982 ST->hasAtomicBufferGlobalPkAddF16NoRtnInsts() ||
983 ST->hasAtomicBufferGlobalPkAddF16Insts();
984 bool HasAtomicDsPkAdd16Insts = ST->hasAtomicDsPkAdd16Insts();
985 addRulesForGOpcs(OpcList: {G_ATOMICRMW_FADD})
986 .Any(Init: {.Predicate: {DivS32, P0, S32}, .OperandMapping: {{Vgpr32}, {VgprP0, Vgpr32}}})
987 .Any(Init: {.Predicate: {DivS64, P0, S64}, .OperandMapping: {{Vgpr64}, {VgprP0, Vgpr64}}})
988 .Any(Init: {.Predicate: {DivS32, P1, S32}, .OperandMapping: {{Vgpr32}, {VgprP1, Vgpr32}}})
989 .Any(Init: {.Predicate: {DivS64, P1, S64}, .OperandMapping: {{Vgpr64}, {VgprP1, Vgpr64}}})
990 .Any(Init: {.Predicate: {DivS32, P3, S32}, .OperandMapping: {{Vgpr32}, {VgprP3, Vgpr32}}})
991 .Any(Init: {.Predicate: {DivS64, P3, S64}, .OperandMapping: {{Vgpr64}, {VgprP3, Vgpr64}}})
992 .Any(Init: {.Predicate: {DivV2S16, P0, V2S16}, .OperandMapping: {{VgprV2S16}, {VgprP0, VgprV2S16}}},
993 STPred: HasAtomicFlatPkAdd16Insts)
994 .Any(Init: {.Predicate: {DivV2S16, P1, V2S16}, .OperandMapping: {{VgprV2S16}, {VgprP1, VgprV2S16}}},
995 STPred: HasAtomicBufferGlobalPkAddF16Insts)
996 .Any(Init: {.Predicate: {DivV2S16, P3, V2S16}, .OperandMapping: {{VgprV2S16}, {VgprP3, VgprV2S16}}},
997 STPred: HasAtomicDsPkAdd16Insts);
998
999 addRulesForGOpcs(OpcList: {G_ATOMIC_CMPXCHG})
1000 .Any(Init: {.Predicate: {DivS32, P2}, .OperandMapping: {{Vgpr32}, {VgprP2, Vgpr32, Vgpr32}}})
1001 .Any(Init: {.Predicate: {DivS64, P2}, .OperandMapping: {{Vgpr64}, {VgprP2, Vgpr64, Vgpr64}}})
1002 .Any(Init: {.Predicate: {DivS32, P3}, .OperandMapping: {{Vgpr32}, {VgprP3, Vgpr32, Vgpr32}}})
1003 .Any(Init: {.Predicate: {DivS64, P3}, .OperandMapping: {{Vgpr64}, {VgprP3, Vgpr64, Vgpr64}}});
1004
1005 addRulesForGOpcs(OpcList: {G_AMDGPU_ATOMIC_CMPXCHG})
1006 .Any(Init: {.Predicate: {DivS32, P0}, .OperandMapping: {{Vgpr32}, {VgprP0, VgprV2S32}}})
1007 .Any(Init: {.Predicate: {DivS32, P1}, .OperandMapping: {{Vgpr32}, {VgprP1, VgprV2S32}}})
1008 .Any(Init: {.Predicate: {DivS64, P0}, .OperandMapping: {{Vgpr64}, {VgprP0, VgprV2S64}}})
1009 .Any(Init: {.Predicate: {DivS64, P1}, .OperandMapping: {{Vgpr64}, {VgprP1, VgprV2S64}}});
1010
1011 addRulesForGOpcs(OpcList: {G_AMDGPU_BUFFER_ATOMIC_CMPSWAP}, FastTypes: Standard)
1012 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32},
1013 {Vgpr32, Vgpr32, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1014 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64},
1015 {Vgpr64, Vgpr64, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
1016
1017 addRulesForGOpcs(OpcList: {G_AMDGPU_BUFFER_ATOMIC_ADD, G_AMDGPU_BUFFER_ATOMIC_AND,
1018 G_AMDGPU_BUFFER_ATOMIC_DEC, G_AMDGPU_BUFFER_ATOMIC_FMAX,
1019 G_AMDGPU_BUFFER_ATOMIC_FMIN, G_AMDGPU_BUFFER_ATOMIC_INC,
1020 G_AMDGPU_BUFFER_ATOMIC_OR, G_AMDGPU_BUFFER_ATOMIC_SMAX,
1021 G_AMDGPU_BUFFER_ATOMIC_SMIN, G_AMDGPU_BUFFER_ATOMIC_SUB,
1022 G_AMDGPU_BUFFER_ATOMIC_SWAP, G_AMDGPU_BUFFER_ATOMIC_UMAX,
1023 G_AMDGPU_BUFFER_ATOMIC_UMIN, G_AMDGPU_BUFFER_ATOMIC_XOR},
1024 FastTypes: Standard)
1025 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1026 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
1027
1028 bool hasSMRDx3 = ST->hasScalarDwordx3Loads();
1029 bool hasSMRDSmall = ST->hasScalarSubwordLoads();
1030 bool usesTrue16 = ST->useRealTrue16Insts();
1031
1032 Predicate isAlign16([](const MachineInstr &MI) -> bool {
1033 return (*MI.memoperands_begin())->getAlign() >= Align(16);
1034 });
1035
1036 Predicate isAlign4([](const MachineInstr &MI) -> bool {
1037 return (*MI.memoperands_begin())->getAlign() >= Align(4);
1038 });
1039
1040 Predicate isAtomicMMO([](const MachineInstr &MI) -> bool {
1041 return (*MI.memoperands_begin())->isAtomic();
1042 });
1043
1044 Predicate isUniMMO([](const MachineInstr &MI) -> bool {
1045 return AMDGPU::isUniformMMO(MMO: *MI.memoperands_begin());
1046 });
1047
1048 Predicate isConst([](const MachineInstr &MI) -> bool {
1049 // Address space in MMO be different then address space on pointer.
1050 const MachineMemOperand *MMO = *MI.memoperands_begin();
1051 const unsigned AS = MMO->getAddrSpace();
1052 return AS == AMDGPUAS::CONSTANT_ADDRESS ||
1053 AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT;
1054 });
1055
1056 Predicate isVolatileMMO([](const MachineInstr &MI) -> bool {
1057 return (*MI.memoperands_begin())->isVolatile();
1058 });
1059
1060 Predicate isInvMMO([](const MachineInstr &MI) -> bool {
1061 return (*MI.memoperands_begin())->isInvariant();
1062 });
1063
1064 Predicate isNoClobberMMO([](const MachineInstr &MI) -> bool {
1065 return (*MI.memoperands_begin())->getFlags() & MONoClobber;
1066 });
1067
1068 Predicate isNaturalAligned([](const MachineInstr &MI) -> bool {
1069 const MachineMemOperand *MMO = *MI.memoperands_begin();
1070 return MMO->getAlign() >= Align(MMO->getSize().getValue());
1071 });
1072
1073 Predicate is8Or16BitMMO([](const MachineInstr &MI) -> bool {
1074 const MachineMemOperand *MMO = *MI.memoperands_begin();
1075 const unsigned MemSize = 8 * MMO->getSize().getValue();
1076 return MemSize == 16 || MemSize == 8;
1077 });
1078
1079 Predicate is32BitMMO([](const MachineInstr &MI) -> bool {
1080 const MachineMemOperand *MMO = *MI.memoperands_begin();
1081 return 8 * MMO->getSize().getValue() == 32;
1082 });
1083
1084 auto isUL = !isAtomicMMO && isUniMMO && (isConst || !isVolatileMMO) &&
1085 (isConst || isInvMMO || isNoClobberMMO);
1086
1087 // clang-format off
1088 // TODO: S32Dst, 16-bit any-extending load should not appear on True16 targets
1089 addRulesForGOpcs(OpcList: {G_LOAD})
1090 // flat, addrspace(0), never uniform - flat_load
1091 .Any(Init: {.Predicate: {DivS16, P0}, .OperandMapping: {{Vgpr16}, {VgprP0}}}, STPred: usesTrue16)
1092 .Any(Init: {.Predicate: {DivB32, P0}, .OperandMapping: {{VgprB32}, {VgprP0}}}) // 32-bit load, 8-bit and 16-bit any-extending load
1093 .Any(Init: {.Predicate: {DivB64, P0}, .OperandMapping: {{VgprB64}, {VgprP0}}})
1094 .Any(Init: {.Predicate: {DivB96, P0}, .OperandMapping: {{VgprB96}, {VgprP0}}})
1095 .Any(Init: {.Predicate: {DivB128, P0}, .OperandMapping: {{VgprB128}, {VgprP0}}})
1096
1097 // global, addrspace(1)
1098 // divergent - global_load
1099 .Any(Init: {.Predicate: {DivS16, P1}, .OperandMapping: {{Vgpr16}, {VgprP1}}}, STPred: usesTrue16)
1100 .Any(Init: {.Predicate: {DivB32, P1}, .OperandMapping: {{VgprB32}, {VgprP1}}}) //32-bit load, 8-bit and 16-bit any-extending load
1101 .Any(Init: {.Predicate: {DivB64, P1}, .OperandMapping: {{VgprB64}, {VgprP1}}})
1102 .Any(Init: {.Predicate: {DivB96, P1}, .OperandMapping: {{VgprB96}, {VgprP1}}})
1103 .Any(Init: {.Predicate: {DivB128, P1}, .OperandMapping: {{VgprB128}, {VgprP1}}})
1104 .Any(Init: {.Predicate: {DivB256, P1}, .OperandMapping: {{VgprB256}, {VgprP1}, SplitLoad}})
1105 .Any(Init: {.Predicate: {DivB512, P1}, .OperandMapping: {{VgprB512}, {VgprP1}, SplitLoad}})
1106
1107 // uniform - s_load
1108 .Any(Init: {.Predicate: {{UniS16, P1}, isNaturalAligned && isUL}, .OperandMapping: {{Sgpr32Trunc}, {SgprP1}}}, STPred: usesTrue16 && hasSMRDSmall) // s16 load
1109 .Any(Init: {.Predicate: {{UniS16, P1}, isAlign4 && isUL}, .OperandMapping: {{Sgpr32Trunc}, {SgprP1}, WidenMMOToS32}}, STPred: usesTrue16 && !hasSMRDSmall) // s16 load to 32-bit load
1110 .Any(Init: {.Predicate: {{UniB32, P1}, isNaturalAligned && isUL}, .OperandMapping: {{SgprB32}, {SgprP1}}}, STPred: hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load
1111 // TODO: SplitLoad when !isNaturalAligned && isUL and target hasSMRDSmall
1112 .Any(Init: {.Predicate: {{UniB32, P1}, is8Or16BitMMO && isAlign4 && isUL}, .OperandMapping: {{SgprB32}, {SgprP1}, WidenMMOToS32}}, STPred: !hasSMRDSmall) //8-bit and 16-bit any-extending load to 32-bit load
1113 .Any(Init: {.Predicate: {{UniB32, P1}, is32BitMMO && isAlign4 && isUL}, .OperandMapping: {{SgprB32}, {SgprP1}}}) //32-bit load
1114 .Any(Init: {.Predicate: {{UniB64, P1}, isAlign4 && isUL}, .OperandMapping: {{SgprB64}, {SgprP1}}})
1115 .Any(Init: {.Predicate: {{UniB96, P1}, isAlign16 && isUL}, .OperandMapping: {{SgprB96}, {SgprP1}, WidenLoad}}, STPred: !hasSMRDx3)
1116 .Any(Init: {.Predicate: {{UniB96, P1}, isAlign4 && !isAlign16 && isUL}, .OperandMapping: {{SgprB96}, {SgprP1}, SplitLoad}}, STPred: !hasSMRDx3)
1117 .Any(Init: {.Predicate: {{UniB96, P1}, isAlign4 && isUL}, .OperandMapping: {{SgprB96}, {SgprP1}}}, STPred: hasSMRDx3)
1118 .Any(Init: {.Predicate: {{UniB128, P1}, isAlign4 && isUL}, .OperandMapping: {{SgprB128}, {SgprP1}}})
1119 .Any(Init: {.Predicate: {{UniB256, P1}, isAlign4 && isUL}, .OperandMapping: {{SgprB256}, {SgprP1}}})
1120 .Any(Init: {.Predicate: {{UniB512, P1}, isAlign4 && isUL}, .OperandMapping: {{SgprB512}, {SgprP1}}})
1121
1122 // Uniform via global or buffer load, for example volatile or non-aligned
1123 // uniform load. Not using standard {{UniInVgprTy}, {VgprP1}} since it is
1124 // selected as global_load, use SgprP1 for pointer instead to match
1125 // patterns without flat-for-global, default for GFX7 and older.
1126 // -> +flat-for-global + {{UniInVgprTy}, {SgprP1}} - global_load
1127 // -> -flat-for-global + {{UniInVgprTy}, {SgprP1}} - buffer_load
1128 .Any(Init: {.Predicate: {{UniS16, P1}, !isNaturalAligned || !isUL}, .OperandMapping: {{UniInVgprS16}, {SgprP1}}}, STPred: usesTrue16 && hasSMRDSmall) // s16 load
1129 .Any(Init: {.Predicate: {{UniS16, P1}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprS16}, {SgprP1}}}, STPred: usesTrue16 && !hasSMRDSmall) // s16 load
1130 .Any(Init: {.Predicate: {{UniB32, P1}, !isNaturalAligned || !isUL}, .OperandMapping: {{UniInVgprB32}, {SgprP1}}}, STPred: hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load
1131 .Any(Init: {.Predicate: {{UniB32, P1}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB32}, {SgprP1}}}, STPred: !hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load
1132 .Any(Init: {.Predicate: {{UniB64, P1}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB64}, {SgprP1}}})
1133 .Any(Init: {.Predicate: {{UniB96, P1}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB96}, {SgprP1}}})
1134 .Any(Init: {.Predicate: {{UniB128, P1}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB128}, {SgprP1}}})
1135 .Any(Init: {.Predicate: {{UniB256, P1}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB256}, {SgprP1}, SplitLoad}})
1136 .Any(Init: {.Predicate: {{UniB512, P1}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB512}, {SgprP1}, SplitLoad}})
1137
1138 // local, addrspace(3) - ds_load
1139 .Any(Init: {.Predicate: {DivS16, P3}, .OperandMapping: {{Vgpr16}, {VgprP3}}}, STPred: usesTrue16)
1140 .Any(Init: {.Predicate: {DivB32, P3}, .OperandMapping: {{VgprB32}, {VgprP3}}}) // 32-bit load, 8-bit and 16-bit any-extending load
1141 .Any(Init: {.Predicate: {DivB64, P3}, .OperandMapping: {{VgprB64}, {VgprP3}}})
1142 .Any(Init: {.Predicate: {DivB96, P3}, .OperandMapping: {{VgprB96}, {VgprP3}}})
1143 .Any(Init: {.Predicate: {DivB128, P3}, .OperandMapping: {{VgprB128}, {VgprP3}}})
1144
1145 .Any(Init: {.Predicate: {UniS16, P3}, .OperandMapping: {{UniInVgprS16}, {SgprP3}}}, STPred: usesTrue16) // 16-bit load
1146 .Any(Init: {.Predicate: {UniB32, P3}, .OperandMapping: {{UniInVgprB32}, {VgprP3}}}) // 32-bit load, 8-bit and 16-bit any-extending load
1147 .Any(Init: {.Predicate: {UniB64, P3}, .OperandMapping: {{UniInVgprB64}, {VgprP3}}})
1148 .Any(Init: {.Predicate: {UniB96, P3}, .OperandMapping: {{UniInVgprB96}, {VgprP3}}})
1149 .Any(Init: {.Predicate: {UniB128, P3}, .OperandMapping: {{UniInVgprB128}, {VgprP3}}})
1150
1151 // constant, addrspace(4)
1152 // divergent - global_load
1153 .Any(Init: {.Predicate: {DivS16, P4}, .OperandMapping: {{Vgpr16}, {VgprP4}}}, STPred: usesTrue16)
1154 .Any(Init: {.Predicate: {DivB32, P4}, .OperandMapping: {{VgprB32}, {VgprP4}}}) //32-bit load, 8-bit and 16-bit any-extending load
1155 .Any(Init: {.Predicate: {DivB64, P4}, .OperandMapping: {{VgprB64}, {VgprP4}}})
1156 .Any(Init: {.Predicate: {DivB96, P4}, .OperandMapping: {{VgprB96}, {VgprP4}}})
1157 .Any(Init: {.Predicate: {DivB128, P4}, .OperandMapping: {{VgprB128}, {VgprP4}}})
1158 .Any(Init: {.Predicate: {DivB256, P4}, .OperandMapping: {{VgprB256}, {VgprP4}, SplitLoad}})
1159 .Any(Init: {.Predicate: {DivB512, P4}, .OperandMapping: {{VgprB512}, {VgprP4}, SplitLoad}})
1160
1161 // uniform - s_load
1162 .Any(Init: {.Predicate: {{UniS16, P4}, isNaturalAligned && isUL}, .OperandMapping: {{Sgpr32Trunc}, {SgprP4}}}, STPred: usesTrue16 && hasSMRDSmall) // s16 load
1163 .Any(Init: {.Predicate: {{UniS16, P4}, isAlign4 && isUL}, .OperandMapping: {{Sgpr32Trunc}, {SgprP4}, WidenMMOToS32}}, STPred: usesTrue16 && !hasSMRDSmall) // s16 load to 32-bit load
1164 .Any(Init: {.Predicate: {{UniB32, P4}, isNaturalAligned && isUL}, .OperandMapping: {{SgprB32}, {SgprP4}}}, STPred: hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load
1165 .Any(Init: {.Predicate: {{UniB32, P4}, is8Or16BitMMO && isAlign4 && isUL}, .OperandMapping: {{SgprB32}, {SgprP4}, WidenMMOToS32}}, STPred: !hasSMRDSmall) //8-bit and 16-bit any-extending load to 32-bit load
1166 .Any(Init: {.Predicate: {{UniB32, P4}, is32BitMMO && isAlign4 && isUL}, .OperandMapping: {{SgprB32}, {SgprP4}}}) //32-bit load
1167 .Any(Init: {.Predicate: {{UniB64, P4}, isAlign4 && isUL}, .OperandMapping: {{SgprB64}, {SgprP4}}})
1168 .Any(Init: {.Predicate: {{UniB96, P4}, isAlign16 && isUL}, .OperandMapping: {{SgprB96}, {SgprP4}, WidenLoad}}, STPred: !hasSMRDx3)
1169 .Any(Init: {.Predicate: {{UniB96, P4}, isAlign4 && !isAlign16 && isUL}, .OperandMapping: {{SgprB96}, {SgprP4}, SplitLoad}}, STPred: !hasSMRDx3)
1170 .Any(Init: {.Predicate: {{UniB96, P4}, isAlign4 && isUL}, .OperandMapping: {{SgprB96}, {SgprP4}}}, STPred: hasSMRDx3)
1171 .Any(Init: {.Predicate: {{UniB128, P4}, isAlign4 && isUL}, .OperandMapping: {{SgprB128}, {SgprP4}}})
1172 .Any(Init: {.Predicate: {{UniB256, P4}, isAlign4 && isUL}, .OperandMapping: {{SgprB256}, {SgprP4}}})
1173 .Any(Init: {.Predicate: {{UniB512, P4}, isAlign4 && isUL}, .OperandMapping: {{SgprB512}, {SgprP4}}})
1174
1175 // uniform in vgpr - global_load or buffer_load
1176 .Any(Init: {.Predicate: {{UniS16, P4}, !isNaturalAligned || !isUL}, .OperandMapping: {{UniInVgprS16}, {SgprP4}}}, STPred: usesTrue16 && hasSMRDSmall) // s16 load
1177 .Any(Init: {.Predicate: {{UniS16, P4}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprS16}, {SgprP4}}}, STPred: usesTrue16 && !hasSMRDSmall) // s16 load
1178 .Any(Init: {.Predicate: {{UniB32, P4}, !isNaturalAligned || !isUL}, .OperandMapping: {{UniInVgprB32}, {SgprP4}}}, STPred: hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load
1179 .Any(Init: {.Predicate: {{UniB32, P4}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB32}, {SgprP4}}}, STPred: !hasSMRDSmall) //32-bit load, 8-bit and 16-bit any-extending load
1180 .Any(Init: {.Predicate: {{UniB64, P4}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB64}, {SgprP4}}})
1181 .Any(Init: {.Predicate: {{UniB96, P4}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB96}, {SgprP4}}})
1182 .Any(Init: {.Predicate: {{UniB128, P4}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB128}, {SgprP4}}})
1183 .Any(Init: {.Predicate: {{UniB256, P4}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB256}, {SgprP4}, SplitLoad}})
1184 .Any(Init: {.Predicate: {{UniB512, P4}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprB512}, {SgprP4}, SplitLoad}})
1185
1186 // private, addrspace(5), never uniform - scratch_load
1187 .Any(Init: {.Predicate: {DivS16, P5}, .OperandMapping: {{Vgpr16}, {VgprP5}}}, STPred: usesTrue16)
1188 .Any(Init: {.Predicate: {DivB32, P5}, .OperandMapping: {{VgprB32}, {VgprP5}}}) // 32-bit load, 8-bit and 16-bit any-extending load
1189 .Any(Init: {.Predicate: {DivB64, P5}, .OperandMapping: {{VgprB64}, {VgprP5}}})
1190 .Any(Init: {.Predicate: {DivB96, P5}, .OperandMapping: {{VgprB96}, {VgprP5}}})
1191 .Any(Init: {.Predicate: {DivB128, P5}, .OperandMapping: {{VgprB128}, {VgprP5}}})
1192
1193 .Any(Init: {.Predicate: {DivS32, Ptr128}, .OperandMapping: {{Vgpr32}, {VgprPtr128}}});
1194
1195
1196 addRulesForGOpcs(OpcList: {G_ZEXTLOAD, G_SEXTLOAD}) // i8 and i16 zeroextending loads
1197 .Any(Init: {.Predicate: {DivS32, P0}, .OperandMapping: {{Vgpr32}, {VgprP0}}})
1198 .Any(Init: {.Predicate: {DivS16, P0}, .OperandMapping: {{Vgpr16}, {VgprP0}}}, STPred: usesTrue16)
1199
1200 .Any(Init: {.Predicate: {DivS32, P1}, .OperandMapping: {{Vgpr32}, {VgprP1}}})
1201 .Any(Init: {.Predicate: {DivS16, P1}, .OperandMapping: {{Vgpr16}, {VgprP1}}}, STPred: usesTrue16)
1202 .Any(Init: {.Predicate: {{UniS16, P1}, isUL}, .OperandMapping: {{Sgpr32Trunc}, {SgprP1}}}, STPred: usesTrue16 && hasSMRDSmall)
1203 .Any(Init: {.Predicate: {{UniS16, P1}, !isUL}, .OperandMapping: {{UniInVgprS16}, {SgprP1}}}, STPred: usesTrue16 && hasSMRDSmall)
1204 .Any(Init: {.Predicate: {UniS16, P1}, .OperandMapping: {{UniInVgprS16}, {SgprP1}}}, STPred: usesTrue16 && !hasSMRDSmall)
1205 .Any(Init: {.Predicate: {{UniS32, P1}, isAlign4 && isUL}, .OperandMapping: {{Sgpr32}, {SgprP1}, WidenMMOToS32}}, STPred: !hasSMRDSmall)
1206 .Any(Init: {.Predicate: {{UniS32, P1}, isNaturalAligned && isUL}, .OperandMapping: {{Sgpr32}, {SgprP1}}}, STPred: hasSMRDSmall)
1207 .Any(Init: {.Predicate: {{UniS32, P1}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprS32}, {SgprP1}}}, STPred: !hasSMRDSmall)
1208 .Any(Init: {.Predicate: {{UniS32, P1}, !isNaturalAligned || !isUL}, .OperandMapping: {{UniInVgprS32}, {SgprP1}}}, STPred: hasSMRDSmall)
1209
1210 .Any(Init: {.Predicate: {DivS32, P3}, .OperandMapping: {{Vgpr32}, {VgprP3}}})
1211 .Any(Init: {.Predicate: {DivS16, P3}, .OperandMapping: {{Vgpr16}, {VgprP3}}}, STPred: usesTrue16)
1212 .Any(Init: {.Predicate: {UniS16, P3}, .OperandMapping: {{UniInVgprS16}, {SgprP3}}}, STPred: usesTrue16)
1213 .Any(Init: {.Predicate: {UniS32, P3}, .OperandMapping: {{UniInVgprS32}, {VgprP3}}})
1214
1215 .Any(Init: {.Predicate: {DivS32, P4}, .OperandMapping: {{Vgpr32}, {VgprP4}}})
1216 .Any(Init: {.Predicate: {DivS16, P4}, .OperandMapping: {{Vgpr16}, {VgprP4}}}, STPred: usesTrue16)
1217 .Any(Init: {.Predicate: {{UniS16, P4}, isUL}, .OperandMapping: {{Sgpr32Trunc}, {SgprP4}}}, STPred: usesTrue16 && hasSMRDSmall)
1218 .Any(Init: {.Predicate: {UniS16, P4}, .OperandMapping: {{UniInVgprS16}, {SgprP4}}}, STPred: usesTrue16 && !hasSMRDSmall)
1219 .Any(Init: {.Predicate: {{UniS32, P4}, isAlign4 && isUL}, .OperandMapping: {{Sgpr32}, {SgprP4}, WidenMMOToS32}}, STPred: !hasSMRDSmall)
1220 .Any(Init: {.Predicate: {{UniS32, P4}, isNaturalAligned && isUL}, .OperandMapping: {{Sgpr32}, {SgprP4}}}, STPred: hasSMRDSmall)
1221 .Any(Init: {.Predicate: {{UniS32, P4}, !isAlign4 || !isUL}, .OperandMapping: {{UniInVgprS32}, {SgprP4}}}, STPred: !hasSMRDSmall)
1222 .Any(Init: {.Predicate: {{UniS32, P4}, !isNaturalAligned || !isUL}, .OperandMapping: {{UniInVgprS32}, {SgprP4}}}, STPred: hasSMRDSmall)
1223
1224 .Any(Init: {.Predicate: {DivS32, P5}, .OperandMapping: {{Vgpr32}, {VgprP5}}})
1225 .Any(Init: {.Predicate: {DivS16, P5}, .OperandMapping: {{Vgpr16}, {VgprP5}}}, STPred: usesTrue16);
1226
1227 addRulesForGOpcs(OpcList: {G_STORE})
1228 // addrspace(0)
1229 .Any(Init: {.Predicate: {S16, P0}, .OperandMapping: {{}, {Vgpr16, VgprP0}}}, STPred: usesTrue16) // 16-bit store
1230 .Any(Init: {.Predicate: {B32, P0}, .OperandMapping: {{}, {VgprB32, VgprP0}}}) // 32-bit store, 8-bit and 16-bit truncating store
1231 .Any(Init: {.Predicate: {B64, P0}, .OperandMapping: {{}, {VgprB64, VgprP0}}})
1232 .Any(Init: {.Predicate: {B96, P0}, .OperandMapping: {{}, {VgprB96, VgprP0}}})
1233 .Any(Init: {.Predicate: {B128, P0}, .OperandMapping: {{}, {VgprB128, VgprP0}}})
1234
1235 // addrspace(1), there are no stores to addrspace(4)
1236 // For targets:
1237 // - with "+flat-for-global" - global_store
1238 // - without(-flat-for-global) - buffer_store addr64
1239 .Any(Init: {.Predicate: {S16, DivP1}, .OperandMapping: {{}, {Vgpr16, VgprP1}}}, STPred: usesTrue16) // 16-bit store
1240 .Any(Init: {.Predicate: {B32, DivP1}, .OperandMapping: {{}, {VgprB32, VgprP1}}}) // 32-bit store, 8-bit and 16-bit truncating store
1241 .Any(Init: {.Predicate: {B64, DivP1}, .OperandMapping: {{}, {VgprB64, VgprP1}}})
1242 .Any(Init: {.Predicate: {B96, DivP1}, .OperandMapping: {{}, {VgprB96, VgprP1}}})
1243 .Any(Init: {.Predicate: {B128, DivP1}, .OperandMapping: {{}, {VgprB128, VgprP1}}})
1244
1245 // For UniP1, use sgpr ptr to match flat-for-global patterns. Targets:
1246 // - with "+flat-for-global" - global_store for both sgpr and vgpr ptr
1247 // - without(-flat-for-global) - need sgpr ptr to select buffer_store
1248 .Any(Init: {.Predicate: {S16, UniP1}, .OperandMapping: {{}, {Vgpr16, SgprP1}}}, STPred: usesTrue16) // 16-bit store
1249 .Any(Init: {.Predicate: {B32, UniP1}, .OperandMapping: {{}, {VgprB32, SgprP1}}}) // 32-bit store, 8-bit and 16-bit truncating store
1250 .Any(Init: {.Predicate: {B64, UniP1}, .OperandMapping: {{}, {VgprB64, SgprP1}}})
1251 .Any(Init: {.Predicate: {B96, UniP1}, .OperandMapping: {{}, {VgprB96, SgprP1}}})
1252 .Any(Init: {.Predicate: {B128, UniP1}, .OperandMapping: {{}, {VgprB128, SgprP1}}})
1253
1254 // addrspace(3) and addrspace(5)
1255 .Any(Init: {.Predicate: {S16, Ptr32}, .OperandMapping: {{}, {Vgpr16, VgprPtr32}}}, STPred: usesTrue16) // 16-bit store
1256 .Any(Init: {.Predicate: {B32, Ptr32}, .OperandMapping: {{}, {VgprB32, VgprPtr32}}}) // 32-bit store, 8-bit and 16-bit truncating store
1257 .Any(Init: {.Predicate: {B64, Ptr32}, .OperandMapping: {{}, {VgprB64, VgprPtr32}}})
1258 .Any(Init: {.Predicate: {B96, Ptr32}, .OperandMapping: {{}, {VgprB96, VgprPtr32}}})
1259 .Any(Init: {.Predicate: {B128, Ptr32}, .OperandMapping: {{}, {VgprB128, VgprPtr32}}});
1260
1261 // clang-format on
1262
1263 addRulesForGOpcs(OpcList: {G_AMDGPU_BUFFER_LOAD, G_AMDGPU_BUFFER_LOAD_FORMAT,
1264 G_AMDGPU_TBUFFER_LOAD_FORMAT},
1265 FastTypes: StandardB)
1266 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1267 .Uni(Ty: B32, RuleApplyIDs: {{UniInVgprB32}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1268 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1269 .Uni(Ty: B64, RuleApplyIDs: {{UniInVgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1270 .Div(Ty: B96, RuleApplyIDs: {{VgprB96}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1271 .Uni(Ty: B96, RuleApplyIDs: {{UniInVgprB96}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1272 .Div(Ty: B128, RuleApplyIDs: {{VgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1273 .Uni(Ty: B128, RuleApplyIDs: {{UniInVgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
1274
1275 addRulesForGOpcs(OpcList: {G_AMDGPU_BUFFER_LOAD_USHORT, G_AMDGPU_BUFFER_LOAD_UBYTE,
1276 G_AMDGPU_BUFFER_LOAD_SSHORT, G_AMDGPU_BUFFER_LOAD_SBYTE},
1277 FastTypes: StandardB)
1278 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1279 .Uni(Ty: B32, RuleApplyIDs: {{UniInVgprB32}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
1280
1281 addRulesForGOpcs(
1282 OpcList: {G_AMDGPU_BUFFER_LOAD_UBYTE_TFE, G_AMDGPU_BUFFER_LOAD_USHORT_TFE},
1283 FastTypes: StandardB)
1284 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1285 .Uni(Ty: B64, RuleApplyIDs: {{UniInVgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
1286
1287 addRulesForGOpcs(OpcList: {G_AMDGPU_BUFFER_LOAD_TFE, G_AMDGPU_BUFFER_LOAD_FORMAT_TFE,
1288 G_AMDGPU_BUFFER_LOAD_FORMAT_D16_TFE},
1289 FastTypes: StandardB)
1290 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1291 .Uni(Ty: B64, RuleApplyIDs: {{UniInVgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1292 .Div(Ty: B96, RuleApplyIDs: {{VgprB96}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1293 .Uni(Ty: B96, RuleApplyIDs: {{UniInVgprB96}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1294 .Div(Ty: B128, RuleApplyIDs: {{VgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1295 .Uni(Ty: B128, RuleApplyIDs: {{UniInVgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1296 .Any(Init: {.Predicate: {DivB160}, .OperandMapping: {{VgprB160}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}})
1297 .Any(Init: {.Predicate: {UniB160},
1298 .OperandMapping: {{UniInVgprB160}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}});
1299
1300 addRulesForGOpcs(
1301 OpcList: {G_AMDGPU_BUFFER_LOAD_FORMAT_D16, G_AMDGPU_TBUFFER_LOAD_FORMAT_D16},
1302 FastTypes: StandardB)
1303 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1304 .Uni(Ty: B32, RuleApplyIDs: {{UniInVgprB32}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1305 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1306 .Uni(Ty: B64, RuleApplyIDs: {{UniInVgprB64}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1307 .Div(Ty: B128, RuleApplyIDs: {{VgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}})
1308 .Uni(Ty: B128, RuleApplyIDs: {{UniInVgprB128}, {SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}});
1309
1310 addRulesForGOpcs(OpcList: {G_AMDGPU_S_BUFFER_LOAD})
1311 // waterfall expansion is part of S_BUF_to_BUF
1312 .Any(Init: {.Predicate: {UniB32}, .OperandMapping: {{SgprB32}, {SgprV4S32, Sgpr32}}})
1313 .Any(Init: {.Predicate: {DivB32, UniV4S32, DivB32},
1314 .OperandMapping: {{VgprB32}, {SgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1315 .Any(Init: {.Predicate: {DivB32, DivV4S32, UniB32},
1316 .OperandMapping: {{VgprB32}, {VgprV4S32, Sgpr32}, S_BUF_to_BUF}})
1317 .Any(Init: {.Predicate: {DivB32, DivV4S32, DivB32},
1318 .OperandMapping: {{VgprB32}, {VgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1319
1320 .Any(Init: {.Predicate: {UniB64}, .OperandMapping: {{SgprB64}, {SgprV4S32, Sgpr32}}})
1321 .Any(Init: {.Predicate: {DivB64, UniV4S32, DivB32},
1322 .OperandMapping: {{VgprB64}, {SgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1323 .Any(Init: {.Predicate: {DivB64, DivV4S32, UniB32},
1324 .OperandMapping: {{VgprB64}, {VgprV4S32, Sgpr32}, S_BUF_to_BUF}})
1325 .Any(Init: {.Predicate: {DivB64, DivV4S32, DivB32},
1326 .OperandMapping: {{VgprB64}, {VgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1327
1328 .Any(Init: {.Predicate: {UniB96}, .OperandMapping: {{SgprB96}, {SgprV4S32, Sgpr32}}})
1329 .Any(Init: {.Predicate: {DivB96, UniV4S32, DivB32},
1330 .OperandMapping: {{VgprB96}, {SgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1331 .Any(Init: {.Predicate: {DivB96, DivV4S32, UniB32},
1332 .OperandMapping: {{VgprB96}, {VgprV4S32, Sgpr32}, S_BUF_to_BUF}})
1333 .Any(Init: {.Predicate: {DivB96, DivV4S32, DivB32},
1334 .OperandMapping: {{VgprB96}, {VgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1335
1336 .Any(Init: {.Predicate: {UniB128}, .OperandMapping: {{SgprB128}, {SgprV4S32, Sgpr32}}})
1337 .Any(Init: {.Predicate: {DivB128, UniV4S32, DivB32},
1338 .OperandMapping: {{VgprB128}, {SgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1339 .Any(Init: {.Predicate: {DivB128, DivV4S32, UniB32},
1340 .OperandMapping: {{VgprB128}, {VgprV4S32, Sgpr32}, S_BUF_to_BUF}})
1341 .Any(Init: {.Predicate: {DivB128, DivV4S32, DivB32},
1342 .OperandMapping: {{VgprB128}, {VgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1343
1344 .Any(Init: {.Predicate: {UniB256}, .OperandMapping: {{SgprB256}, {SgprV4S32, Sgpr32}}})
1345 .Any(Init: {.Predicate: {DivB256, UniV4S32, DivB32},
1346 .OperandMapping: {{VgprB256}, {SgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1347 .Any(Init: {.Predicate: {DivB256, DivV4S32, UniB32},
1348 .OperandMapping: {{VgprB256}, {VgprV4S32, Sgpr32}, S_BUF_to_BUF}})
1349 .Any(Init: {.Predicate: {DivB256, DivV4S32, DivB32},
1350 .OperandMapping: {{VgprB256}, {VgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1351
1352 .Any(Init: {.Predicate: {UniB512}, .OperandMapping: {{SgprB512}, {SgprV4S32, Sgpr32}}})
1353 .Any(Init: {.Predicate: {DivB512, UniV4S32, DivB32},
1354 .OperandMapping: {{VgprB512}, {SgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1355 .Any(Init: {.Predicate: {DivB512, DivV4S32, UniB32},
1356 .OperandMapping: {{VgprB512}, {VgprV4S32, Sgpr32}, S_BUF_to_BUF}})
1357 .Any(Init: {.Predicate: {DivB512, DivV4S32, DivB32},
1358 .OperandMapping: {{VgprB512}, {VgprV4S32, Vgpr32}, S_BUF_to_BUF}});
1359
1360 addRulesForGOpcs(OpcList: {G_AMDGPU_S_BUFFER_LOAD_SBYTE, G_AMDGPU_S_BUFFER_LOAD_UBYTE,
1361 G_AMDGPU_S_BUFFER_LOAD_SSHORT,
1362 G_AMDGPU_S_BUFFER_LOAD_USHORT})
1363 .Any(Init: {.Predicate: {UniS32, UniV4S32, UniS32}, .OperandMapping: {{Sgpr32}, {SgprV4S32, Sgpr32}}})
1364 .Any(Init: {.Predicate: {DivS32, UniV4S32, DivS32},
1365 .OperandMapping: {{Vgpr32}, {SgprV4S32, Vgpr32}, S_BUF_to_BUF}})
1366 .Any(Init: {.Predicate: {DivS32, DivV4S32, UniS32},
1367 .OperandMapping: {{Vgpr32}, {VgprV4S32, Sgpr32}, S_BUF_to_BUF}})
1368 .Any(Init: {.Predicate: {DivS32, DivV4S32, DivS32},
1369 .OperandMapping: {{Vgpr32}, {VgprV4S32, Vgpr32}, S_BUF_to_BUF}});
1370
1371 addRulesForGOpcs(OpcList: {G_AMDGPU_BUFFER_STORE, G_AMDGPU_BUFFER_STORE_BYTE,
1372 G_AMDGPU_BUFFER_STORE_SHORT, G_AMDGPU_BUFFER_STORE_FORMAT,
1373 G_AMDGPU_BUFFER_STORE_FORMAT_D16,
1374 G_AMDGPU_TBUFFER_STORE_FORMAT,
1375 G_AMDGPU_TBUFFER_STORE_FORMAT_D16})
1376 .Any(Init: {.Predicate: {B32}, .OperandMapping: {{}, {VgprB32, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}})
1377 .Any(Init: {.Predicate: {B64}, .OperandMapping: {{}, {VgprB64, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}})
1378 .Any(Init: {.Predicate: {B96}, .OperandMapping: {{}, {VgprB96, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}})
1379 .Any(Init: {.Predicate: {B128}, .OperandMapping: {{}, {VgprB128, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}});
1380
1381 // Buffer atomics: resource descriptor + scalar offset are SGPR, data and
1382 // address components are VGPR.
1383 //
1384 // Operand order (SIInstructions.td BufferAtomicGenericInstruction):
1385 // dst = op vdata, rsrc, vindex, voffset, soffset, offset_imm, cachepolicy,
1386 // idxen_imm
1387 addRulesForGOpcs(OpcList: {G_AMDGPU_BUFFER_ATOMIC_FADD})
1388 .Any(Init: {.Predicate: {S32, S32, V4S32, S32, S32, S32},
1389 .OperandMapping: {{Vgpr32}, {Vgpr32, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}})
1390 .Any(Init: {.Predicate: {S64, S64, V4S32, S32, S32, S32},
1391 .OperandMapping: {{Vgpr64}, {Vgpr64, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}})
1392 .Any(Init: {.Predicate: {V2S16, V2S16, V4S32, S32, S32, S32},
1393 .OperandMapping: {{VgprV2S16},
1394 {VgprV2S16, SgprV4S32_WF, Vgpr32, Vgpr32, Sgpr32_WF}}});
1395
1396 addRulesForGOpcs(OpcList: {G_PTR_ADD})
1397 .Any(Init: {.Predicate: {UniPtr32}, .OperandMapping: {{SgprPtr32}, {SgprPtr32, Sgpr32}}})
1398 .Any(Init: {.Predicate: {DivPtr32}, .OperandMapping: {{VgprPtr32}, {VgprPtr32, Vgpr32}}})
1399 .Any(Init: {.Predicate: {UniPtr64}, .OperandMapping: {{SgprPtr64}, {SgprPtr64, Sgpr64}}})
1400 .Any(Init: {.Predicate: {DivPtr64}, .OperandMapping: {{VgprPtr64}, {VgprPtr64, Vgpr64}}});
1401
1402 addRulesForGOpcs(OpcList: {G_INTTOPTR})
1403 .Any(Init: {.Predicate: {UniPtr32}, .OperandMapping: {{SgprPtr32}, {Sgpr32}}})
1404 .Any(Init: {.Predicate: {DivPtr32}, .OperandMapping: {{VgprPtr32}, {Vgpr32}}})
1405 .Any(Init: {.Predicate: {UniPtr64}, .OperandMapping: {{SgprPtr64}, {Sgpr64}}})
1406 .Any(Init: {.Predicate: {DivPtr64}, .OperandMapping: {{VgprPtr64}, {Vgpr64}}})
1407 .Any(Init: {.Predicate: {UniPtr128}, .OperandMapping: {{SgprPtr128}, {Sgpr128}}})
1408 .Any(Init: {.Predicate: {DivPtr128}, .OperandMapping: {{VgprPtr128}, {Vgpr128}}});
1409
1410 addRulesForGOpcs(OpcList: {G_PTRTOINT})
1411 .Any(Init: {.Predicate: {UniS32}, .OperandMapping: {{Sgpr32}, {SgprPtr32}}})
1412 .Any(Init: {.Predicate: {DivS32}, .OperandMapping: {{Vgpr32}, {VgprPtr32}}})
1413 .Any(Init: {.Predicate: {UniS64}, .OperandMapping: {{Sgpr64}, {SgprPtr64}}})
1414 .Any(Init: {.Predicate: {DivS64}, .OperandMapping: {{Vgpr64}, {VgprPtr64}}})
1415 .Any(Init: {.Predicate: {UniS128}, .OperandMapping: {{Sgpr128}, {SgprPtr128}}})
1416 .Any(Init: {.Predicate: {DivS128}, .OperandMapping: {{Vgpr128}, {VgprPtr128}}});
1417
1418 // FIXME: Update llvm/test/CodeGen/AMDGPU/ptrmask.ll to use GlobalISel.
1419 // Currently crashes on P8 (buffer resource) tests due to legalizer issue.
1420 addRulesForGOpcs(OpcList: {G_PTRMASK})
1421 .Any(Init: {.Predicate: {UniP1}, .OperandMapping: {{SgprP1}, {SgprP1, Sgpr64}}})
1422 .Any(Init: {.Predicate: {DivP1}, .OperandMapping: {{VgprP1}, {VgprP1, Vgpr64}}})
1423 .Any(Init: {.Predicate: {UniP3}, .OperandMapping: {{SgprP3}, {SgprP3, Sgpr32}}})
1424 .Any(Init: {.Predicate: {DivP3}, .OperandMapping: {{VgprP3}, {VgprP3, Vgpr32}}});
1425
1426 addRulesForGOpcs(OpcList: {G_DYN_STACKALLOC})
1427 .Any(Init: {.Predicate: {UniP5, UniS32}, .OperandMapping: {{SgprP5}, {Sgpr32}, DynStackAlloc}})
1428 .Any(Init: {.Predicate: {UniP5, DivS32}, .OperandMapping: {{SgprP5}, {Vgpr32}, DynStackAlloc}});
1429
1430 addRulesForGOpcs(OpcList: {G_ABS}, FastTypes: Standard)
1431 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr32Trunc}, {Sgpr32SExt}})
1432 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16}, AbsToNegMax})
1433 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32}})
1434 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32}, AbsToNegMax})
1435 .Uni(Ty: V2S16, RuleApplyIDs: {{SgprV2S16}, {SgprV2S16}, AbsToS32})
1436 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16}, AbsToNegMax});
1437
1438 addRulesForGOpcs(OpcList: {G_BITREVERSE}, FastTypes: Standard)
1439 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32}})
1440 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32}})
1441 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {Sgpr64}})
1442 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64}});
1443
1444 addRulesForGOpcs(OpcList: {G_AMDGPU_FFBH_U32, G_AMDGPU_FFBL_B32, G_CTLZ_ZERO_POISON,
1445 G_CTTZ_ZERO_POISON})
1446 .Any(Init: {.Predicate: {UniS32, S32}, .OperandMapping: {{Sgpr32}, {Sgpr32}}})
1447 .Any(Init: {.Predicate: {DivS32, S32}, .OperandMapping: {{Vgpr32}, {Vgpr32}}})
1448 .Any(Init: {.Predicate: {UniS32, S64}, .OperandMapping: {{Sgpr32}, {Sgpr64}}})
1449 .Any(Init: {.Predicate: {DivS32, S64}, .OperandMapping: {{Vgpr32}, {Vgpr64}, SplitBitCount64To32}});
1450
1451 addRulesForGOpcs(OpcList: {G_CTPOP})
1452 .Any(Init: {.Predicate: {UniS32, S32}, .OperandMapping: {{Sgpr32}, {Sgpr32}}})
1453 .Any(Init: {.Predicate: {DivS32, S32}, .OperandMapping: {{Vgpr32}, {Vgpr32}}})
1454 .Any(Init: {.Predicate: {UniS32, S64}, .OperandMapping: {{Sgpr32}, {Sgpr64}}})
1455 .Any(Init: {.Predicate: {DivS32, S64}, .OperandMapping: {{Vgpr32}, {Vgpr64}, CtPop64To32}});
1456
1457 addRulesForGOpcs(OpcList: {G_FENCE}).Any(Init: {.Predicate: {{}}, .OperandMapping: {{}, {}}});
1458
1459 addRulesForGOpcs(OpcList: {G_READSTEADYCOUNTER, G_READCYCLECOUNTER}, FastTypes: Standard)
1460 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {}});
1461
1462 addRulesForGOpcs(OpcList: {G_GET_ROUNDING}, FastTypes: Standard)
1463 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {}, LowerGetRounding});
1464
1465 addRulesForGOpcs(OpcList: {G_SET_ROUNDING}, FastTypes: Standard)
1466 .Uni(Ty: S32, RuleApplyIDs: {{}, {SgprB32_ReadFirstLane}, LowerSetRounding})
1467 .Div(Ty: S32, RuleApplyIDs: {{}, {SgprB32_ReadFirstLane}, LowerSetRounding});
1468
1469 addRulesForGOpcs(OpcList: {G_WRITE_REGISTER})
1470 .Any(Init: {.Predicate: {_, B32}, .OperandMapping: {{}, {None, SgprB32_ReadFirstLane}}})
1471 .Any(Init: {.Predicate: {_, B64}, .OperandMapping: {{}, {None, SgprB64_ReadFirstLane}}});
1472
1473 addRulesForGOpcs(OpcList: {G_BLOCK_ADDR}).Any(Init: {.Predicate: {UniP0}, .OperandMapping: {{SgprP0}, {}}});
1474
1475 addRulesForGOpcs(OpcList: {G_GLOBAL_VALUE})
1476 .Any(Init: {.Predicate: {UniP0}, .OperandMapping: {{SgprP0}, {}}})
1477 .Any(Init: {.Predicate: {UniP1}, .OperandMapping: {{SgprP1}, {}}})
1478 .Any(Init: {.Predicate: {UniP3}, .OperandMapping: {{SgprP3}, {}}})
1479 .Any(Init: {.Predicate: {UniP4}, .OperandMapping: {{SgprP4}, {}}})
1480 .Any(Init: {.Predicate: {UniP8}, .OperandMapping: {{SgprP8}, {}}});
1481
1482 addRulesForGOpcs(OpcList: {G_AMDGPU_WAVE_ADDRESS}).Any(Init: {.Predicate: {UniP5}, .OperandMapping: {{SgprP5}, {}}});
1483
1484 addRulesForGOpcs(OpcList: {G_AMDGPU_SPONENTRY}, FastTypes: Standard).Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {}});
1485
1486 addRulesForGOpcs(OpcList: {G_SI_CALL})
1487 .Any(Init: {.Predicate: {_, UniP0}, .OperandMapping: {{None}, {SgprP0}}})
1488 .Any(Init: {.Predicate: {_, DivP0}, .OperandMapping: {{None}, {SgprP0Call_WF}}})
1489 .Any(Init: {.Predicate: {_, UniP4}, .OperandMapping: {{None}, {SgprP4}}})
1490 .Any(Init: {.Predicate: {_, DivP4}, .OperandMapping: {{None}, {SgprP4Call_WF}}});
1491
1492 bool hasSALUFloat = ST->hasSALUFloatInsts();
1493
1494 addRulesForGOpcs(OpcList: {G_FADD, G_FMUL, G_STRICT_FADD, G_STRICT_FMUL}, FastTypes: Standard)
1495 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16, Vgpr16}}, STPred: !hasSALUFloat)
1496 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {Sgpr16, Sgpr16}}, STPred: hasSALUFloat)
1497 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
1498 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}}, STPred: hasSALUFloat)
1499 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32}}, STPred: !hasSALUFloat)
1500 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
1501 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64, Vgpr64}})
1502 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr64}})
1503 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16}})
1504 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
1505 .Any(Init: {.Predicate: {UniV2S32}, .OperandMapping: {{UniInVgprV2S32}, {VgprV2S32, VgprV2S32}}})
1506 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {VgprV2S32, VgprV2S32}}})
1507 .Any(Init: {.Predicate: {UniV2S64}, .OperandMapping: {{UniInVgprV2S64}, {VgprV2S64, VgprV2S64}}})
1508 .Any(Init: {.Predicate: {DivV2S64}, .OperandMapping: {{VgprV2S64}, {VgprV2S64, VgprV2S64}}});
1509
1510 addRulesForGOpcs(OpcList: {G_FSUB, G_STRICT_FSUB}, FastTypes: Standard)
1511 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
1512 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
1513 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {Sgpr16, Sgpr16}}, STPred: hasSALUFloat)
1514 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16, Vgpr16}}, STPred: !hasSALUFloat)
1515 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}}, STPred: hasSALUFloat)
1516 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32}}, STPred: !hasSALUFloat)
1517 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16}})
1518 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}});
1519
1520 addRulesForGOpcs(OpcList: {G_FMAD}, FastTypes: Standard)
1521 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16, Vgpr16, Vgpr16}})
1522 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16, Vgpr16}})
1523 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32, Vgpr32}})
1524 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32, Vgpr32}});
1525
1526 addRulesForGOpcs(OpcList: {G_FLDEXP, G_STRICT_FLDEXP}, FastTypes: Standard)
1527 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32}})
1528 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
1529 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16, Vgpr16}})
1530 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
1531 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64, Vgpr32}})
1532 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr32}});
1533
1534 addRulesForGOpcs(OpcList: {G_FMA, G_STRICT_FMA}, FastTypes: Standard)
1535 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {Sgpr16, Sgpr16, Sgpr16}}, STPred: hasSALUFloat)
1536 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16, Vgpr16, Vgpr16}}, STPred: !hasSALUFloat)
1537 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16, Vgpr16}})
1538 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32, Sgpr32}}, STPred: hasSALUFloat)
1539 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32, Vgpr32}}, STPred: !hasSALUFloat)
1540 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32, Vgpr32}})
1541 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64, Vgpr64, Vgpr64}})
1542 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr64, Vgpr64}})
1543 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16, VgprV2S16}})
1544 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16, VgprV2S16}})
1545 .Any(Init: {.Predicate: {UniV2S32}, .OperandMapping: {{UniInVgprV2S32}, {VgprV2S32, VgprV2S32, VgprV2S32}}})
1546 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {VgprV2S32, VgprV2S32, VgprV2S32}}})
1547 .Any(Init: {.Predicate: {UniV2S64}, .OperandMapping: {{UniInVgprV2S64}, {VgprV2S64, VgprV2S64, VgprV2S64}}})
1548 .Any(Init: {.Predicate: {DivV2S64}, .OperandMapping: {{VgprV2S64}, {VgprV2S64, VgprV2S64, VgprV2S64}}});
1549
1550 addRulesForGOpcs(OpcList: {G_AMDGPU_FMED3}, FastTypes: Standard)
1551 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16, Vgpr16, Vgpr16}})
1552 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16, Vgpr16}})
1553 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32, Vgpr32}})
1554 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32, Vgpr32}});
1555
1556 // TODO: This opcode is generated from the i64->i16 signed clamped pattern in
1557 // the PreLegalizerCombiner. Move the combine to RegBankCombiner to keep more
1558 // instructions on SALU.
1559 addRulesForGOpcs(OpcList: {G_AMDGPU_SMED3}, FastTypes: Standard)
1560 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32, Vgpr32}})
1561 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32, Vgpr32}});
1562
1563 // FNEG and FABS are either folded as source modifiers or can be selected as
1564 // bitwise XOR and AND with Mask. XOR and AND are available on SALU but for
1565 // targets without SALU float we still select them as VGPR since there would
1566 // be no real sgpr use.
1567 addRulesForGOpcs(OpcList: {G_FNEG, G_FABS}, FastTypes: Standard)
1568 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16}}, STPred: !hasSALUFloat)
1569 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {Sgpr16}}, STPred: hasSALUFloat)
1570 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16}})
1571 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32}}, STPred: !hasSALUFloat)
1572 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32}}, STPred: hasSALUFloat)
1573 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32}})
1574 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64}})
1575 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64}})
1576 .Uni(Ty: V2S16, RuleApplyIDs: {{SgprV2S16}, {SgprV2S16}})
1577 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16}})
1578 .Any(Init: {.Predicate: {UniV2S32}, .OperandMapping: {{UniInVgprV2S32}, {VgprV2S32}}})
1579 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {VgprV2S32}}});
1580
1581 addRulesForGOpcs(OpcList: {G_FCANONICALIZE}, FastTypes: Standard)
1582 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32}})
1583 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32}})
1584 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16}})
1585 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16}})
1586 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64}})
1587 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64}})
1588 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {VgprV2S16}})
1589 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16}})
1590 .Any(Init: {.Predicate: {UniV2S32}, .OperandMapping: {{UniInVgprV2S32}, {VgprV2S32}}})
1591 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {VgprV2S32}}})
1592 .Any(Init: {.Predicate: {UniV2S64}, .OperandMapping: {{UniInVgprV2S64}, {VgprV2S64}}})
1593 .Any(Init: {.Predicate: {DivV2S64}, .OperandMapping: {{VgprV2S64}, {VgprV2S64}}});
1594
1595 bool hasPST = ST->hasPseudoScalarTrans();
1596 // Only 16-bit G_FSQRT operations are legal. F32/F64 is handled with a custom
1597 // legalization in the legalizer.
1598 addRulesForGOpcs(OpcList: {G_FSQRT})
1599 .Any(Init: {.Predicate: {DivS16}, .OperandMapping: {{Vgpr16}, {Vgpr16}}})
1600 .Any(Init: {.Predicate: {UniBF16}, .OperandMapping: {{UniInVgprS16}, {Vgpr16}}})
1601 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{Sgpr16}, {Sgpr16}}}, STPred: hasPST)
1602 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{UniInVgprS16}, {Vgpr16}}}, STPred: !hasPST);
1603
1604 addRulesForGOpcs(OpcList: {G_FPTOUI, G_FPTOSI, G_FPTOUI_SAT, G_FPTOSI_SAT})
1605 .Any(Init: {.Predicate: {UniS16, S16}, .OperandMapping: {{UniInVgprS16}, {Vgpr16}}})
1606 .Any(Init: {.Predicate: {DivS16, S16}, .OperandMapping: {{Vgpr16}, {Vgpr16}}})
1607 .Any(Init: {.Predicate: {UniS32, S16}, .OperandMapping: {{Sgpr32}, {Sgpr16}}}, STPred: hasSALUFloat)
1608 .Any(Init: {.Predicate: {UniS32, S16}, .OperandMapping: {{UniInVgprS32}, {Vgpr16}}}, STPred: !hasSALUFloat)
1609 .Any(Init: {.Predicate: {DivS32, S16}, .OperandMapping: {{Vgpr32}, {Vgpr16}}})
1610 .Any(Init: {.Predicate: {UniS16, S32}, .OperandMapping: {{Sgpr16}, {Sgpr32}}}, STPred: hasSALUFloat)
1611 .Any(Init: {.Predicate: {UniS16, S32}, .OperandMapping: {{UniInVgprS16}, {Vgpr32}}}, STPred: !hasSALUFloat)
1612 .Any(Init: {.Predicate: {DivS16, S32}, .OperandMapping: {{Vgpr16}, {Vgpr32}}})
1613 .Any(Init: {.Predicate: {UniS32, S32}, .OperandMapping: {{Sgpr32}, {Sgpr32}}}, STPred: hasSALUFloat)
1614 .Any(Init: {.Predicate: {UniS32, S32}, .OperandMapping: {{UniInVgprS32}, {Vgpr32}}}, STPred: !hasSALUFloat)
1615 .Any(Init: {.Predicate: {DivS32, S32}, .OperandMapping: {{Vgpr32}, {Vgpr32}}})
1616 .Any(Init: {.Predicate: {UniS32, S64}, .OperandMapping: {{UniInVgprS32}, {Vgpr64}}})
1617 .Any(Init: {.Predicate: {DivS32, S64}, .OperandMapping: {{Vgpr32}, {Vgpr64}}})
1618 .Any(Init: {.Predicate: {UniV2S16, V2S32}, .OperandMapping: {{UniInVgprV2S16}, {SgprV2S32}}})
1619 .Any(Init: {.Predicate: {DivV2S16, V2S32}, .OperandMapping: {{VgprV2S16}, {VgprV2S32}}});
1620
1621 addRulesForGOpcs(OpcList: {G_UITOFP, G_SITOFP})
1622 .Any(Init: {.Predicate: {UniS16, S16}, .OperandMapping: {{UniInVgprS16}, {Vgpr16}}})
1623 .Any(Init: {.Predicate: {DivS16, S16}, .OperandMapping: {{Vgpr16}, {Vgpr16}}})
1624 .Any(Init: {.Predicate: {UniS16, S32}, .OperandMapping: {{Sgpr16}, {Sgpr32}}}, STPred: hasSALUFloat)
1625 .Any(Init: {.Predicate: {UniS16, S32}, .OperandMapping: {{UniInVgprS16}, {Vgpr32}}}, STPred: !hasSALUFloat)
1626 .Any(Init: {.Predicate: {DivS16, S32}, .OperandMapping: {{Vgpr16}, {Vgpr32}}})
1627 .Any(Init: {.Predicate: {UniS32, S32}, .OperandMapping: {{Sgpr32}, {Sgpr32}}}, STPred: hasSALUFloat)
1628 .Any(Init: {.Predicate: {UniS32, S32}, .OperandMapping: {{UniInVgprS32}, {Vgpr32}}}, STPred: !hasSALUFloat)
1629 .Any(Init: {.Predicate: {DivS32, S32}, .OperandMapping: {{Vgpr32}, {Vgpr32}}})
1630 .Any(Init: {.Predicate: {UniS64, S32}, .OperandMapping: {{UniInVgprS64}, {Vgpr32}}})
1631 .Any(Init: {.Predicate: {DivS64, S32}, .OperandMapping: {{Vgpr64}, {Vgpr32}}});
1632
1633 addRulesForGOpcs(OpcList: {G_AMDGPU_S_BUFFER_PREFETCH})
1634 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {SgprV4S32_ReadFirstLane, Imm, SgprB32_ReadFirstLane}}});
1635
1636 Predicate IsDataPF([](const MachineInstr &MI) -> bool {
1637 // prefetch cache type: 0 == instruction (I$) prefetch, 1 == data prefetch.
1638 return MI.getOperand(i: 3).getImm() != 0;
1639 });
1640
1641 bool HasSMemPF = ST->hasSafeSmemPrefetch();
1642 bool HasVMemPF = ST->hasVmemPrefInsts();
1643 addRulesForGOpcs(OpcList: {G_PREFETCH})
1644 // Safe smem prefetch keeps both data and instruction prefetch.
1645 .Any(Init: {.Predicate: {UniPtr64}, .OperandMapping: {{}, {SgprPtr64}}}, STPred: HasSMemPF)
1646 // Vmem prefetch keeps data prefetch only.
1647 .Any(Init: {.Predicate: {{UniPtr64}, IsDataPF}, .OperandMapping: {{}, {SgprPtr64}}}, STPred: !HasSMemPF && HasVMemPF)
1648 .Any(Init: {.Predicate: {{UniPtr64}, IsDataPF}, .OperandMapping: {{}, {}, DeletePrefetch}},
1649 STPred: !HasSMemPF && !HasVMemPF)
1650 .Any(Init: {.Predicate: {{UniPtr64}, !IsDataPF}, .OperandMapping: {{}, {}, DeletePrefetch}}, STPred: !HasSMemPF)
1651
1652 .Any(Init: {.Predicate: {{DivPtr64}, IsDataPF}, .OperandMapping: {{}, {VgprPtr64}}}, STPred: HasVMemPF)
1653 .Any(Init: {.Predicate: {{DivPtr64}, IsDataPF}, .OperandMapping: {{}, {}, DeletePrefetch}}, STPred: !HasVMemPF)
1654 .Any(Init: {.Predicate: {{DivPtr64}, !IsDataPF}, .OperandMapping: {{}, {}, DeletePrefetch}})
1655
1656 .Any(Init: {.Predicate: {P3}, .OperandMapping: {{}, {}, DeletePrefetch}})
1657 .Any(Init: {.Predicate: {P5}, .OperandMapping: {{}, {}, DeletePrefetch}})
1658 .Any(Init: {.Predicate: {UniP6}, .OperandMapping: {{}, {SgprP6}}}, STPred: HasSMemPF)
1659 .Any(Init: {.Predicate: {UniP6}, .OperandMapping: {{}, {}, DeletePrefetch}}, STPred: !HasSMemPF);
1660
1661 addRulesForGOpcs(OpcList: {G_FPEXT})
1662 .Any(Init: {.Predicate: {DivS32, S16}, .OperandMapping: {{Vgpr32}, {Vgpr16}}})
1663 .Any(Init: {.Predicate: {UniS64, S32}, .OperandMapping: {{UniInVgprS64}, {Vgpr32}}})
1664 .Any(Init: {.Predicate: {DivS64, S32}, .OperandMapping: {{Vgpr64}, {Vgpr32}}})
1665 .Any(Init: {.Predicate: {UniS32, S16}, .OperandMapping: {{Sgpr32}, {Sgpr16}}}, STPred: hasSALUFloat)
1666 .Any(Init: {.Predicate: {UniS32, S16}, .OperandMapping: {{UniInVgprS32}, {Vgpr16}}}, STPred: !hasSALUFloat);
1667
1668 addRulesForGOpcs(OpcList: {G_AMDGPU_CVT_PK_I16_I32}, FastTypes: Standard)
1669 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {Vgpr32, Vgpr32}})
1670 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {Vgpr32, Vgpr32}});
1671
1672 addRulesForGOpcs(OpcList: {G_AMDGPU_FMIN_LEGACY, G_AMDGPU_FMAX_LEGACY}, FastTypes: Standard)
1673 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32}})
1674 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}});
1675
1676 bool hasSALUMinimumMaximumInsts = ST->hasSALUMinimumMaximumInsts();
1677
1678 addRulesForGOpcs(OpcList: {G_FMINIMUM, G_FMAXIMUM}, FastTypes: Standard)
1679 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {Sgpr16, Sgpr16}}, STPred: hasSALUMinimumMaximumInsts)
1680 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16, Vgpr16}}, STPred: !hasSALUMinimumMaximumInsts)
1681 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
1682 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}}, STPred: hasSALUMinimumMaximumInsts)
1683 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32}}, STPred: !hasSALUMinimumMaximumInsts)
1684 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
1685 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64, Vgpr64}})
1686 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr64}})
1687 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16}})
1688 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}});
1689
1690 addRulesForGOpcs(OpcList: {G_FMINNUM_IEEE, G_FMAXNUM_IEEE, G_FMINNUM, G_FMAXNUM,
1691 G_FMINIMUMNUM, G_FMAXIMUMNUM},
1692 FastTypes: Standard)
1693 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16, Vgpr16}})
1694 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32, Vgpr32}})
1695 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64, Vgpr64}})
1696 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64, Vgpr64}})
1697 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {VgprV2S16, VgprV2S16}})
1698 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {VgprV2S16, VgprV2S16}})
1699 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {Sgpr16, Sgpr16}}, STPred: hasSALUFloat)
1700 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16, Vgpr16}}, STPred: !hasSALUFloat)
1701 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32, Sgpr32}}, STPred: hasSALUFloat)
1702 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32, Vgpr32}}, STPred: !hasSALUFloat);
1703
1704 addRulesForGOpcs(OpcList: {G_FPTRUNC})
1705 .Any(Init: {.Predicate: {DivS16, S32}, .OperandMapping: {{Vgpr16}, {Vgpr32}}})
1706 .Any(Init: {.Predicate: {UniS32, S64}, .OperandMapping: {{UniInVgprS32}, {Vgpr64}}})
1707 .Any(Init: {.Predicate: {DivS32, S64}, .OperandMapping: {{Vgpr32}, {Vgpr64}}})
1708 .Any(Init: {.Predicate: {UniV2S16, V2S32}, .OperandMapping: {{UniInVgprV2S16}, {VgprV2S32}}})
1709 .Any(Init: {.Predicate: {DivV2S16, V2S32}, .OperandMapping: {{VgprV2S16}, {VgprV2S32}}})
1710 .Any(Init: {.Predicate: {UniS16, S32}, .OperandMapping: {{Sgpr16}, {Sgpr32}}}, STPred: hasSALUFloat)
1711 .Any(Init: {.Predicate: {UniS16, S32}, .OperandMapping: {{UniInVgprS16}, {Vgpr32}}}, STPred: !hasSALUFloat);
1712
1713 addRulesForGOpcs(OpcList: {G_INTRINSIC_FPTRUNC_ROUND})
1714 .Any(Init: {.Predicate: {UniS16, S32}, .OperandMapping: {{Sgpr16}, {Sgpr32}}}, STPred: hasSALUFloat)
1715 .Any(Init: {.Predicate: {UniS16, S32}, .OperandMapping: {{UniInVgprS16}, {Vgpr32}}}, STPred: !hasSALUFloat)
1716 .Any(Init: {.Predicate: {DivS16, S32}, .OperandMapping: {{Vgpr16}, {Vgpr32}}})
1717 .Any(Init: {.Predicate: {UniS16, S64}, .OperandMapping: {{UniInVgprS16}, {Vgpr64}}})
1718 .Any(Init: {.Predicate: {DivS16, S64}, .OperandMapping: {{Vgpr16}, {Vgpr64}}})
1719 .Any(Init: {.Predicate: {UniS32, S64}, .OperandMapping: {{UniInVgprS32}, {Vgpr64}}})
1720 .Any(Init: {.Predicate: {DivS32, S64}, .OperandMapping: {{Vgpr32}, {Vgpr64}}});
1721
1722 addRulesForGOpcs(OpcList: {G_IS_FPCLASS})
1723 .Any(Init: {.Predicate: {DivS1, S16}, .OperandMapping: {{Vcc}, {Vgpr16}}})
1724 .Any(Init: {.Predicate: {UniS1, S16}, .OperandMapping: {{UniInVcc}, {Vgpr16}}})
1725 .Any(Init: {.Predicate: {DivS1, S32}, .OperandMapping: {{Vcc}, {Vgpr32}}})
1726 .Any(Init: {.Predicate: {UniS1, S32}, .OperandMapping: {{UniInVcc}, {Vgpr32}}})
1727 .Any(Init: {.Predicate: {DivS1, S64}, .OperandMapping: {{Vcc}, {Vgpr64}}})
1728 .Any(Init: {.Predicate: {UniS1, S64}, .OperandMapping: {{UniInVcc}, {Vgpr64}}});
1729
1730 addRulesForGOpcs(OpcList: {G_FCMP}, FastTypes: Standard)
1731 .Any(Init: {.Predicate: {UniS1, _, S16}, .OperandMapping: {{Sgpr32Trunc}, {None, Sgpr16, Sgpr16}}},
1732 STPred: hasSALUFloat)
1733 .Any(Init: {.Predicate: {UniS1, _, S16}, .OperandMapping: {{UniInVcc}, {None, Vgpr16, Vgpr16}}},
1734 STPred: !hasSALUFloat)
1735 .Any(Init: {.Predicate: {DivS1, _, S16}, .OperandMapping: {{Vcc}, {None, Vgpr16, Vgpr16}}})
1736 .Any(Init: {.Predicate: {UniS1, _, S32}, .OperandMapping: {{Sgpr32Trunc}, {None, Sgpr32, Sgpr32}}},
1737 STPred: hasSALUFloat)
1738 .Any(Init: {.Predicate: {UniS1, _, S32}, .OperandMapping: {{UniInVcc}, {None, Vgpr32, Vgpr32}}},
1739 STPred: !hasSALUFloat)
1740 .Any(Init: {.Predicate: {DivS1, _, S32}, .OperandMapping: {{Vcc}, {None, Vgpr32, Vgpr32}}})
1741 .Any(Init: {.Predicate: {UniS1, _, S64}, .OperandMapping: {{UniInVcc}, {None, Vgpr64, Vgpr64}}})
1742 .Any(Init: {.Predicate: {DivS1, _, S64}, .OperandMapping: {{Vcc}, {None, Vgpr64, Vgpr64}}});
1743
1744 addRulesForGOpcs(OpcList: {G_INTRINSIC_ROUNDEVEN, G_FEXP2, G_FLOG2}, FastTypes: Standard)
1745 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16}})
1746 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16}})
1747 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32}})
1748 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32}})
1749 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64}})
1750 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64}});
1751
1752 addRulesForGOpcs(OpcList: {G_INTRINSIC_TRUNC, G_FFLOOR, G_FCEIL}, FastTypes: Standard)
1753 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {Vgpr16}})
1754 .Uni(Ty: S16, RuleApplyIDs: {{Sgpr16}, {Sgpr16}}, STPred: hasSALUFloat)
1755 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {Vgpr16}})
1756 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {Vgpr32}})
1757 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {Sgpr32}}, STPred: hasSALUFloat)
1758 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {Vgpr32}})
1759 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {Vgpr64}})
1760 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {Vgpr64}});
1761
1762 addRulesForGOpcs(OpcList: {G_AMDGPU_GLOBAL_LOAD_MONITOR, G_AMDGPU_FLAT_LOAD_MONITOR},
1763 FastTypes: StandardB)
1764 .Uni(Ty: B32, RuleApplyIDs: {{UniInVgprB32}, {SgprPtr64}})
1765 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {VgprPtr64}})
1766 .Uni(Ty: B64, RuleApplyIDs: {{UniInVgprB64}, {SgprPtr64}})
1767 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {VgprPtr64}})
1768 .Uni(Ty: B128, RuleApplyIDs: {{UniInVgprB128}, {SgprPtr64}})
1769 .Div(Ty: B128, RuleApplyIDs: {{VgprB128}, {VgprPtr64}});
1770
1771 addRulesForGOpcs(OpcList: {G_AMDGPU_WHOLE_WAVE_FUNC_SETUP})
1772 .Any(Init: {.Predicate: {DivS1}, .OperandMapping: {{Vcc}, {}}});
1773
1774 addRulesForGOpcs(OpcList: {G_AMDGPU_WHOLE_WAVE_FUNC_RETURN}).Any(Init: {.Predicate: {}, .OperandMapping: {{}, {Vcc}}});
1775
1776 using namespace Intrinsic;
1777
1778 addRulesForIOpcs(OpcList: {returnaddress}).Any(Init: {.Predicate: {UniP0}, .OperandMapping: {{SgprP0}, {}}});
1779
1780 addRulesForIOpcs(OpcList: {amdgcn_s_bitreplicate}, FastTypes: Standard)
1781 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {IntrId, Sgpr32}})
1782 .Div(Ty: S64, RuleApplyIDs: {{Sgpr64ToVgprDst}, {IntrId, SgprB32_ReadFirstLane}});
1783
1784 addRulesForIOpcs(OpcList: {amdgcn_s_getpc}).Any(Init: {.Predicate: {UniS64, _}, .OperandMapping: {{Sgpr64}, {}}});
1785
1786 addRulesForIOpcs(OpcList: {amdgcn_s_getreg}).Any(Init: {.Predicate: {}, .OperandMapping: {{Sgpr32}, {IntrId}}});
1787
1788 addRulesForIOpcs(OpcList: {amdgcn_s_setreg})
1789 .Any(Init: {.Predicate: {_, _, S32}, .OperandMapping: {{}, {IntrId, Imm, SgprB32_ReadFirstLane}}});
1790
1791 addRulesForIOpcs(OpcList: {amdgcn_s_sendmsg, amdgcn_s_sendmsghalt})
1792 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, Imm, SgprB32_M0}}});
1793
1794 addRulesForIOpcs(OpcList: {amdgcn_s_sendmsg_rtn})
1795 .Any(Init: {.Predicate: {S32}, .OperandMapping: {{Sgpr32}, {}}})
1796 .Any(Init: {.Predicate: {S64}, .OperandMapping: {{Sgpr64}, {}}});
1797
1798 addRulesForIOpcs(OpcList: {amdgcn_s_memrealtime, amdgcn_s_memtime}, FastTypes: Standard)
1799 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {IntrId}});
1800
1801 addRulesForIOpcs(OpcList: {amdgcn_groupstaticsize, amdgcn_pops_exiting_wave_id,
1802 amdgcn_reloc_constant, amdgcn_s_get_waveid_in_workgroup},
1803 FastTypes: Standard)
1804 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {IntrId}});
1805
1806 // Intrinsics with no register operands.
1807 addRulesForIOpcs(OpcList: {amdgcn_asyncmark,
1808 amdgcn_endpgm,
1809 amdgcn_iglp_opt,
1810 amdgcn_init_exec,
1811 amdgcn_s_barrier,
1812 amdgcn_s_barrier_leave,
1813 amdgcn_s_barrier_signal,
1814 amdgcn_s_barrier_wait,
1815 amdgcn_s_decperflevel,
1816 amdgcn_s_incperflevel,
1817 amdgcn_s_monitor_sleep,
1818 amdgcn_s_nop,
1819 amdgcn_s_sethalt,
1820 amdgcn_s_setprio,
1821 amdgcn_s_setprio_inc_wg,
1822 amdgcn_s_sleep,
1823 amdgcn_s_ttracedata_imm,
1824 amdgcn_s_wait_asynccnt,
1825 amdgcn_s_wait_bvhcnt,
1826 amdgcn_s_wait_dscnt,
1827 amdgcn_s_wait_event,
1828 amdgcn_s_wait_event_export_ready,
1829 amdgcn_s_wait_expcnt,
1830 amdgcn_s_wait_kmcnt,
1831 amdgcn_s_wait_loadcnt,
1832 amdgcn_s_wait_samplecnt,
1833 amdgcn_s_wait_storecnt,
1834 amdgcn_s_wait_tensorcnt,
1835 amdgcn_s_waitcnt,
1836 amdgcn_sched_barrier,
1837 amdgcn_sched_group_barrier,
1838 amdgcn_unreachable,
1839 amdgcn_wait_asyncmark,
1840 amdgcn_wave_barrier})
1841 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {}}});
1842
1843 addRulesForIOpcs(OpcList: {amdgcn_init_exec_from_input})
1844 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, Sgpr32}}});
1845
1846 addRulesForIOpcs(OpcList: {amdgcn_s_ttracedata}).Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, SgprB32_M0}}});
1847
1848 addRulesForIOpcs(OpcList: {amdgcn_s_sleep_var})
1849 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, SgprB32_ReadFirstLane}}});
1850
1851 addRulesForIOpcs(OpcList: {amdgcn_s_barrier_join, amdgcn_s_wakeup_barrier})
1852 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, SgprB32_M0}}});
1853
1854 addRulesForIOpcs(OpcList: {amdgcn_s_barrier_signal_var, amdgcn_s_barrier_init})
1855 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, SgprB32_M0, SgprB32_M0}}});
1856
1857 addRulesForIOpcs(OpcList: {amdgcn_s_barrier_signal_isfirst})
1858 .Any(Init: {.Predicate: {UniS1}, .OperandMapping: {{Sgpr32Trunc}, {}}});
1859
1860 addRulesForIOpcs(
1861 OpcList: {amdgcn_s_get_named_barrier_state, amdgcn_s_get_barrier_state}, FastTypes: Standard)
1862 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {IntrId, SgprB32_M0}});
1863
1864 addRulesForIOpcs(OpcList: {amdgcn_flat_prefetch}).Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, VgprP0}}});
1865
1866 addRulesForIOpcs(OpcList: {amdgcn_global_prefetch}).Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, VgprP1}}});
1867
1868 addRulesForIOpcs(OpcList: {amdgcn_s_prefetch_data, amdgcn_s_prefetch_inst})
1869 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, SgprB64_ReadFirstLane, SgprB32_ReadFirstLane}}});
1870
1871 addRulesForIOpcs(OpcList: {amdgcn_class})
1872 .Any(Init: {.Predicate: {UniS1, _, S16}, .OperandMapping: {{UniInVcc}, {IntrId, Vgpr16, Vgpr32}}})
1873 .Any(Init: {.Predicate: {DivS1, _, S16}, .OperandMapping: {{Vcc}, {IntrId, Vgpr16, Vgpr32}}})
1874 .Any(Init: {.Predicate: {UniS1, _, S32}, .OperandMapping: {{UniInVcc}, {IntrId, Vgpr32, Vgpr32}}})
1875 .Any(Init: {.Predicate: {DivS1, _, S32}, .OperandMapping: {{Vcc}, {IntrId, Vgpr32, Vgpr32}}})
1876 .Any(Init: {.Predicate: {UniS1, _, S64}, .OperandMapping: {{UniInVcc}, {IntrId, Vgpr64, Vgpr32}}})
1877 .Any(Init: {.Predicate: {DivS1, _, S64}, .OperandMapping: {{Vcc}, {IntrId, Vgpr64, Vgpr32}}});
1878
1879 // This is "intrinsic lane mask" it was set to i32/i64 in llvm-ir.
1880 addRulesForIOpcs(OpcList: {amdgcn_end_cf})
1881 .Any(Init: {.Predicate: {_, UniS32}, .OperandMapping: {{}, {IntrId, Sgpr32}}})
1882 .Any(Init: {.Predicate: {_, UniS64}, .OperandMapping: {{}, {IntrId, Sgpr64}}});
1883
1884 addRulesForIOpcs(OpcList: {amdgcn_if_break}, FastTypes: Standard)
1885 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {IntrId, Vcc, Sgpr64}})
1886 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {IntrId, Vcc, Sgpr32}});
1887
1888 addRulesForIOpcs(OpcList: {amdgcn_exp})
1889 .Any(Init: {.Predicate: {_, _, _, S32, S32, S32, S32},
1890 .OperandMapping: {{}, {IntrId, Imm, Imm, Vgpr32, Vgpr32, Vgpr32, Vgpr32}}});
1891
1892 addRulesForIOpcs(OpcList: {amdgcn_exp_compr})
1893 .Any(Init: {.Predicate: {_, _, _, V2S16}, .OperandMapping: {{}, {IntrId, Imm, Imm, VgprV2S16, VgprV2S16}}});
1894
1895 addRulesForIOpcs(OpcList: {amdgcn_exp_row})
1896 .Any(Init: {.Predicate: {_, _, _, S32, S32, S32, S32, _, S32},
1897 .OperandMapping: {{},
1898 {IntrId, Imm, Imm, Vgpr32, Vgpr32, Vgpr32, Vgpr32, Imm,
1899 SgprB32_M0}}});
1900
1901 addRulesForIOpcs(OpcList: {amdgcn_lds_direct_load}, FastTypes: StandardB)
1902 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {IntrId, SgprB32_M0}});
1903
1904 addRulesForIOpcs(OpcList: {amdgcn_lds_param_load}, FastTypes: Standard)
1905 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Imm, Imm, SgprB32_M0}});
1906
1907 addRulesForIOpcs(OpcList: {amdgcn_mbcnt_lo, amdgcn_mbcnt_hi}, FastTypes: Standard)
1908 .Div(Ty: S32, RuleApplyIDs: {{}, {Vgpr32, None, Vgpr32, Vgpr32}});
1909
1910 addRulesForIOpcs(OpcList: {amdgcn_readfirstlane})
1911 .Any(Init: {.Predicate: {UniB32, _, DivB32}, .OperandMapping: {{}, {SgprB32, None, VgprB32}}})
1912 // this should not exist in the first place, it is from call lowering
1913 // readfirstlaning just in case register is not in sgpr.
1914 .Any(Init: {.Predicate: {UniS32, _, UniS32}, .OperandMapping: {{}, {Sgpr32, None, Vgpr32}}});
1915
1916 addRulesForIOpcs(OpcList: {amdgcn_readlane}, FastTypes: StandardB)
1917 .Uni(Ty: B32, RuleApplyIDs: {{SgprB32}, {IntrId, VgprB32, SgprB32_ReadFirstLane}});
1918
1919 addRulesForIOpcs(OpcList: {amdgcn_s_quadmask, amdgcn_s_wqm}, FastTypes: StandardB)
1920 .Uni(Ty: B32, RuleApplyIDs: {{SgprB32}, {IntrId, SgprB32_ReadFirstLane}})
1921 .Uni(Ty: B64, RuleApplyIDs: {{SgprB64}, {IntrId, SgprB64_ReadFirstLane}});
1922
1923 addRulesForIOpcs(OpcList: {amdgcn_writelane}, FastTypes: StandardB)
1924 .Div(Ty: B32,
1925 RuleApplyIDs: {{VgprB32},
1926 {IntrId, SgprB32_ReadFirstLane, SgprB32_ReadFirstLane, VgprB32}});
1927
1928 addRulesForIOpcs(OpcList: {amdgcn_add_max_i32, amdgcn_add_max_u32, amdgcn_add_min_i32,
1929 amdgcn_add_min_u32},
1930 FastTypes: Standard)
1931 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
1932 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
1933
1934 addRulesForIOpcs(OpcList: {amdgcn_pk_add_max_i16, amdgcn_pk_add_max_u16,
1935 amdgcn_pk_add_min_i16, amdgcn_pk_add_min_u16},
1936 FastTypes: Standard)
1937 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {IntrId, VgprV2S16, VgprV2S16, VgprV2S16}})
1938 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {IntrId, VgprV2S16, VgprV2S16, VgprV2S16}});
1939
1940 addRulesForIOpcs(OpcList: {amdgcn_permlane16, amdgcn_permlanex16}, FastTypes: StandardB)
1941 .Div(Ty: B32, RuleApplyIDs: {{VgprB32},
1942 {IntrId, VgprB32, VgprB32, SgprB32_ReadFirstLane,
1943 SgprB32_ReadFirstLane}});
1944
1945 addRulesForIOpcs(OpcList: {amdgcn_permlane_bcast, amdgcn_permlane_up,
1946 amdgcn_permlane_down, amdgcn_permlane_xor},
1947 FastTypes: StandardB)
1948 .Div(Ty: B32,
1949 RuleApplyIDs: {{VgprB32},
1950 {IntrId, VgprB32, SgprB32_ReadFirstLane, SgprB32_ReadFirstLane}});
1951
1952 addRulesForIOpcs(OpcList: {amdgcn_permlane_idx_gen}, FastTypes: Standard)
1953 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, SgprB32_ReadFirstLane}});
1954
1955 addRulesForIOpcs(OpcList: {amdgcn_alignbyte, amdgcn_perm}, FastTypes: Standard)
1956 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
1957 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
1958
1959 addRulesForIOpcs(OpcList: {amdgcn_mqsad_pk_u16_u8, amdgcn_qsad_pk_u16_u8}, FastTypes: Standard)
1960 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {IntrId, Vgpr64, Vgpr32, Vgpr64}})
1961 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {IntrId, Vgpr64, Vgpr32, Vgpr64}});
1962
1963 addRulesForIOpcs(OpcList: {amdgcn_mqsad_u32_u8})
1964 .Any(
1965 Init: {.Predicate: {UniV4S32}, .OperandMapping: {{UniInVgprV4S32}, {IntrId, Vgpr64, Vgpr32, VgprV4S32}}})
1966 .Any(Init: {.Predicate: {DivV4S32}, .OperandMapping: {{VgprV4S32}, {IntrId, Vgpr64, Vgpr32, VgprV4S32}}});
1967
1968 addRulesForIOpcs(OpcList: {amdgcn_lerp}, FastTypes: Standard)
1969 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
1970 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
1971
1972 addRulesForIOpcs(
1973 OpcList: {amdgcn_msad_u8, amdgcn_sad_hi_u8, amdgcn_sad_u16, amdgcn_sad_u8},
1974 FastTypes: Standard)
1975 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
1976 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
1977
1978 addRulesForIOpcs(
1979 OpcList: {amdgcn_wave_reduce_add, amdgcn_wave_reduce_and, amdgcn_wave_reduce_fadd,
1980 amdgcn_wave_reduce_fmax, amdgcn_wave_reduce_fmin,
1981 amdgcn_wave_reduce_fsub, amdgcn_wave_reduce_max, amdgcn_wave_reduce_min,
1982 amdgcn_wave_reduce_or, amdgcn_wave_reduce_sub, amdgcn_wave_reduce_umax,
1983 amdgcn_wave_reduce_umin, amdgcn_wave_reduce_xor},
1984 FastTypes: Standard)
1985 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {IntrId, Sgpr32}})
1986 .Div(Ty: S32, RuleApplyIDs: {{Sgpr32ToVgprDst}, {IntrId, VgprB32}})
1987 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {IntrId, Sgpr64}})
1988 .Div(Ty: S64, RuleApplyIDs: {{Sgpr64ToVgprDst}, {IntrId, VgprB64}});
1989
1990 addRulesForIOpcs(OpcList: {amdgcn_wave_shuffle}, FastTypes: Standard)
1991 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32}})
1992 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32}});
1993
1994 addRulesForIOpcs(OpcList: {amdgcn_bitop3, amdgcn_fmad_ftz}, FastTypes: Standard)
1995 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, Vgpr16, Vgpr16, Vgpr16}})
1996 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, Vgpr16, Vgpr16, Vgpr16}})
1997 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
1998 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
1999
2000 addRulesForIOpcs(OpcList: {amdgcn_udot4, amdgcn_sdot4, amdgcn_udot8, amdgcn_sdot8,
2001 amdgcn_dot4_f32_bf8_bf8, amdgcn_dot4_f32_bf8_fp8,
2002 amdgcn_dot4_f32_fp8_fp8, amdgcn_dot4_f32_fp8_bf8},
2003 FastTypes: Standard)
2004 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
2005 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
2006
2007 addRulesForIOpcs(OpcList: {amdgcn_rsq, amdgcn_rsq_clamp})
2008 .Any(Init: {.Predicate: {UniBF16}, .OperandMapping: {{UniInVgprS16}, {IntrId, Vgpr16}}})
2009 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{Sgpr16}, {IntrId, Sgpr16}}}, STPred: hasPST)
2010 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{UniInVgprS16}, {IntrId, Vgpr16}}}, STPred: !hasPST)
2011 .Any(Init: {.Predicate: {DivS16}, .OperandMapping: {{Vgpr16}, {IntrId, Vgpr16}}})
2012 .Any(Init: {.Predicate: {UniS32}, .OperandMapping: {{Sgpr32}, {IntrId, Sgpr32}}}, STPred: hasPST)
2013 .Any(Init: {.Predicate: {UniS32}, .OperandMapping: {{UniInVgprS32}, {IntrId, Vgpr32}}}, STPred: !hasPST)
2014 .Any(Init: {.Predicate: {DivS32}, .OperandMapping: {{Vgpr32}, {IntrId, Vgpr32}}})
2015 .Any(Init: {.Predicate: {UniS64}, .OperandMapping: {{UniInVgprS64}, {IntrId, Vgpr64}}})
2016 .Any(Init: {.Predicate: {DivS64}, .OperandMapping: {{Vgpr64}, {IntrId, Vgpr64}}});
2017
2018 addRulesForIOpcs(OpcList: {amdgcn_mul_u24, amdgcn_mul_i24}, FastTypes: Standard)
2019 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32}})
2020 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32}})
2021 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {IntrId, Vgpr32, Vgpr32}})
2022 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {IntrId, Vgpr32, Vgpr32}});
2023
2024 addRulesForIOpcs(OpcList: {amdgcn_ds_bpermute, amdgcn_ds_bpermute_fi_b32,
2025 amdgcn_ds_permute, amdgcn_fmul_legacy, amdgcn_mulhi_i24,
2026 amdgcn_mulhi_u24},
2027 FastTypes: Standard)
2028 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32}})
2029 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32}});
2030
2031 addRulesForIOpcs(OpcList: {amdgcn_cvt_sr_bf8_f32, amdgcn_cvt_sr_fp8_f32,
2032 amdgcn_cvt_sr_fp8_f32_e5m3, amdgcn_cvt_pk_bf8_f32,
2033 amdgcn_cvt_pk_fp8_f32, amdgcn_cvt_pk_fp8_f32_e5m3,
2034 amdgcn_cvt_pk_u8_f32},
2035 FastTypes: Standard)
2036 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
2037 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
2038
2039 addRulesForIOpcs(OpcList: {amdgcn_cvt_off_f32_i4, amdgcn_cvt_f32_bf8,
2040 amdgcn_cvt_f32_fp8, amdgcn_cvt_f32_fp8_e5m3},
2041 FastTypes: Standard)
2042 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32}})
2043 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32}});
2044
2045 addRulesForIOpcs(OpcList: {amdgcn_cvt_pk_f32_bf8, amdgcn_cvt_pk_f32_fp8})
2046 .Any(Init: {.Predicate: {UniV2S32}, .OperandMapping: {{UniInVgprV2S32}, {IntrId, Vgpr32}}})
2047 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {IntrId, Vgpr32}}});
2048
2049 addRulesForIOpcs(OpcList: {amdgcn_cvt_f16_bf8, amdgcn_cvt_f16_fp8}, FastTypes: Standard)
2050 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, Vgpr32}})
2051 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, Vgpr32}});
2052
2053 addRulesForIOpcs(OpcList: {amdgcn_cvt_pk_f16_bf8, amdgcn_cvt_pk_f16_fp8}, FastTypes: Standard)
2054 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {IntrId, Vgpr16}})
2055 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {IntrId, Vgpr16}});
2056
2057 addRulesForIOpcs(OpcList: {amdgcn_cvt_pk_bf8_f16, amdgcn_cvt_pk_fp8_f16}, FastTypes: Standard)
2058 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, VgprV2S16}})
2059 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, VgprV2S16}});
2060
2061 addRulesForIOpcs(OpcList: {amdgcn_cvt_sr_bf8_f16, amdgcn_cvt_sr_fp8_f16}, FastTypes: Standard)
2062 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr16, Vgpr32, Vgpr32}})
2063 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr16, Vgpr32, Vgpr32}});
2064
2065 addRulesForIOpcs(OpcList: {amdgcn_cvt_sr_pk_f16_f32, amdgcn_cvt_sr_pk_bf16_f32},
2066 FastTypes: Standard)
2067 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
2068 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
2069
2070 addRulesForIOpcs(
2071 OpcList: {amdgcn_cvt_scalef32_sr_fp8_f16, amdgcn_cvt_scalef32_sr_bf8_f16,
2072 amdgcn_cvt_scalef32_sr_fp8_bf16, amdgcn_cvt_scalef32_sr_bf8_bf16})
2073 .Any(Init: {.Predicate: {DivS32}, .OperandMapping: {{Vgpr32}, {IntrId, Vgpr32, Vgpr16, Vgpr32, Vgpr32}}})
2074 .Any(Init: {.Predicate: {UniS32},
2075 .OperandMapping: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr16, Vgpr32, Vgpr32}}});
2076
2077 addRulesForIOpcs(
2078 OpcList: {amdgcn_cvt_scalef32_sr_fp8_f32, amdgcn_cvt_scalef32_sr_bf8_f32})
2079 .Any(Init: {.Predicate: {DivS32}, .OperandMapping: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32, Vgpr32}}})
2080 .Any(Init: {.Predicate: {UniS32},
2081 .OperandMapping: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32, Vgpr32}}});
2082
2083 addRulesForIOpcs(OpcList: {amdgcn_cubesc, amdgcn_cubetc, amdgcn_cubema, amdgcn_cubeid,
2084 amdgcn_fma_legacy},
2085 FastTypes: Standard)
2086 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
2087 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
2088
2089 addRulesForIOpcs(OpcList: {amdgcn_frexp_mant, amdgcn_fract}, FastTypes: Standard)
2090 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, Vgpr16}})
2091 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, Vgpr16}})
2092 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32}})
2093 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32}})
2094 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {IntrId, Vgpr64}})
2095 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {IntrId, Vgpr64}});
2096
2097 addRulesForIOpcs(OpcList: {amdgcn_prng_b32})
2098 .Any(Init: {.Predicate: {UniS32}, .OperandMapping: {{UniInVgprS32}, {IntrId, Vgpr32}}})
2099 .Any(Init: {.Predicate: {DivS32}, .OperandMapping: {{Vgpr32}, {IntrId, Vgpr32}}});
2100
2101 addRulesForIOpcs(
2102 OpcList: {amdgcn_exclusive_scan_sum_i32, amdgcn_exclusive_scan_sum_u32}, FastTypes: Standard)
2103 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Imm}})
2104 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Imm}});
2105
2106 addRulesForIOpcs(
2107 OpcList: {amdgcn_exclusive_scan_xor_b32, amdgcn_exclusive_scan_or_b32,
2108 amdgcn_exclusive_scan_and_b32, amdgcn_exclusive_scan_min_i32,
2109 amdgcn_exclusive_scan_min_u32, amdgcn_exclusive_scan_max_i32,
2110 amdgcn_exclusive_scan_max_u32},
2111 FastTypes: Standard)
2112 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32}})
2113 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32}});
2114
2115 addRulesForIOpcs(
2116 OpcList: {amdgcn_exclusive_scan_min_i16, amdgcn_exclusive_scan_min_u16,
2117 amdgcn_exclusive_scan_max_i16, amdgcn_exclusive_scan_max_u16},
2118 FastTypes: Standard)
2119 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, Vgpr16, Vgpr32}})
2120 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, Vgpr16, Vgpr32}});
2121
2122 addRulesForIOpcs(OpcList: {amdgcn_wave_match_b32})
2123 .Any(Init: {.Predicate: {UniS32}, .OperandMapping: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32}}})
2124 .Any(Init: {.Predicate: {DivS32}, .OperandMapping: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32}}});
2125
2126 addRulesForIOpcs(OpcList: {amdgcn_sffbh}, FastTypes: Standard)
2127 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {IntrId, Sgpr32}})
2128 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32}});
2129
2130 addRulesForIOpcs(OpcList: {amdgcn_ubfe, amdgcn_sbfe}, FastTypes: Standard)
2131 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
2132 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {IntrId, Sgpr32, Sgpr32, Sgpr32}, S_BFE})
2133 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {IntrId, Sgpr64, Sgpr32, Sgpr32}, S_BFE})
2134 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {IntrId, Vgpr64, Vgpr32, Vgpr32}, V_BFE});
2135
2136 addRulesForIOpcs(OpcList: {amdgcn_cvt_pk_i16, amdgcn_cvt_pk_u16, amdgcn_cvt_pknorm_i16,
2137 amdgcn_cvt_pknorm_u16},
2138 FastTypes: Standard)
2139 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {IntrId, Vgpr32, Vgpr32}})
2140 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {IntrId, Vgpr32, Vgpr32}});
2141
2142 addRulesForIOpcs(OpcList: {amdgcn_cvt_pkrtz}, FastTypes: Standard)
2143 .Uni(Ty: V2S16, RuleApplyIDs: {{SgprV2S16}, {IntrId, Sgpr32, Sgpr32}}, STPred: hasSALUFloat)
2144 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {IntrId, Vgpr32, Vgpr32}}, STPred: !hasSALUFloat)
2145 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {IntrId, Vgpr32, Vgpr32}});
2146
2147 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_sr_pk32_bf6_f16,
2148 amdgcn_cvt_scalef32_sr_pk32_fp6_f16,
2149 amdgcn_cvt_scalef32_sr_pk32_bf6_bf16,
2150 amdgcn_cvt_scalef32_sr_pk32_fp6_bf16},
2151 FastTypes: Standard)
2152 .Any(Init: {.Predicate: {DivV6S32}, .OperandMapping: {{VgprV6S32}, {IntrId, VgprV32S16, Vgpr32, Vgpr32}}})
2153 .Any(Init: {.Predicate: {UniV6S32},
2154 .OperandMapping: {{UniInVgprV6S32}, {IntrId, VgprV32S16, Vgpr32, Vgpr32}}});
2155
2156 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_sr_pk32_bf6_f32,
2157 amdgcn_cvt_scalef32_sr_pk32_fp6_f32},
2158 FastTypes: Standard)
2159 .Any(Init: {.Predicate: {DivV6S32}, .OperandMapping: {{VgprV6S32}, {IntrId, VgprV32S32, Vgpr32, Vgpr32}}})
2160 .Any(Init: {.Predicate: {UniV6S32},
2161 .OperandMapping: {{UniInVgprV6S32}, {IntrId, VgprV32S32, Vgpr32, Vgpr32}}});
2162
2163 addRulesForIOpcs(
2164 OpcList: {amdgcn_cvt_scalef32_sr_pk_fp4_f16, amdgcn_cvt_scalef32_sr_pk_fp4_bf16},
2165 FastTypes: Standard)
2166 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, VgprV2S16, Vgpr32, Vgpr32}})
2167 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, VgprV2S16, Vgpr32, Vgpr32}});
2168
2169 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_sr_pk_fp4_f32}, FastTypes: Standard)
2170 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, VgprV2S32, Vgpr32, Vgpr32}})
2171 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, VgprV2S32, Vgpr32, Vgpr32}});
2172
2173 addRulesForIOpcs(
2174 OpcList: {amdgcn_cvt_scalef32_2xpk16_fp6_f32, amdgcn_cvt_scalef32_2xpk16_bf6_f32})
2175 .Any(
2176 Init: {.Predicate: {DivV6S32}, .OperandMapping: {{VgprV6S32}, {IntrId, VgprV16S32, VgprV16S32, Vgpr32}}})
2177 .Any(Init: {.Predicate: {UniV6S32},
2178 .OperandMapping: {{UniInVgprV6S32}, {IntrId, VgprV16S32, VgprV16S32, Vgpr32}}});
2179
2180 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_f16_fp8, amdgcn_cvt_scalef32_f16_bf8},
2181 FastTypes: Standard)
2182 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {IntrId, VgprV2S16, Vgpr32, Vgpr32}})
2183 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {IntrId, VgprV2S16, Vgpr32, Vgpr32}});
2184
2185 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_f32_fp8, amdgcn_cvt_scalef32_f32_bf8},
2186 FastTypes: Standard)
2187 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32}})
2188 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32}});
2189
2190 addRulesForIOpcs(
2191 OpcList: {amdgcn_cvt_scalef32_pk16_bf6_f16, amdgcn_cvt_scalef32_pk16_fp6_f16,
2192 amdgcn_cvt_scalef32_pk16_bf6_bf16, amdgcn_cvt_scalef32_pk16_fp6_bf16},
2193 FastTypes: Standard)
2194 .Any(Init: {.Predicate: {DivV3S32}, .OperandMapping: {{VgprV3S32}, {IntrId, VgprV16S16, Vgpr32}}})
2195 .Any(Init: {.Predicate: {UniV3S32}, .OperandMapping: {{UniInVgprV3S32}, {IntrId, VgprV16S16, Vgpr32}}});
2196
2197 addRulesForIOpcs(
2198 OpcList: {amdgcn_cvt_scalef32_pk16_bf6_f32, amdgcn_cvt_scalef32_pk16_fp6_f32},
2199 FastTypes: Standard)
2200 .Any(Init: {.Predicate: {DivV3S32}, .OperandMapping: {{VgprV3S32}, {IntrId, VgprV16S32, Vgpr32}}})
2201 .Any(Init: {.Predicate: {UniV3S32}, .OperandMapping: {{UniInVgprV3S32}, {IntrId, VgprV16S32, Vgpr32}}});
2202
2203 addRulesForIOpcs(
2204 OpcList: {amdgcn_cvt_scalef32_pk8_bf8_f16, amdgcn_cvt_scalef32_pk8_fp8_f16,
2205 amdgcn_cvt_scalef32_pk8_bf8_bf16, amdgcn_cvt_scalef32_pk8_fp8_bf16},
2206 FastTypes: Standard)
2207 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {IntrId, VgprV8S16, Vgpr32}}})
2208 .Any(Init: {.Predicate: {UniV2S32}, .OperandMapping: {{UniInVgprV2S32}, {IntrId, VgprV8S16, Vgpr32}}});
2209
2210 addRulesForIOpcs(
2211 OpcList: {amdgcn_cvt_scalef32_pk8_bf8_f32, amdgcn_cvt_scalef32_pk8_fp8_f32},
2212 FastTypes: Standard)
2213 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {IntrId, VgprV8S32, Vgpr32}}})
2214 .Any(Init: {.Predicate: {UniV2S32}, .OperandMapping: {{UniInVgprV2S32}, {IntrId, VgprV8S32, Vgpr32}}});
2215
2216 addRulesForIOpcs(
2217 OpcList: {amdgcn_cvt_scalef32_pk8_fp4_f16, amdgcn_cvt_scalef32_pk8_fp4_bf16},
2218 FastTypes: Standard)
2219 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, VgprV8S16, Vgpr32}})
2220 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, VgprV8S16, Vgpr32}});
2221
2222 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_pk8_fp4_f32}, FastTypes: Standard)
2223 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, VgprV8S32, Vgpr32}})
2224 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, VgprV8S32, Vgpr32}});
2225
2226 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_sr_pk16_bf6_f16,
2227 amdgcn_cvt_scalef32_sr_pk16_fp6_f16,
2228 amdgcn_cvt_scalef32_sr_pk16_bf6_bf16,
2229 amdgcn_cvt_scalef32_sr_pk16_fp6_bf16},
2230 FastTypes: Standard)
2231 .Any(Init: {.Predicate: {DivV3S32}, .OperandMapping: {{VgprV3S32}, {IntrId, VgprV16S16, Vgpr32, Vgpr32}}})
2232 .Any(Init: {.Predicate: {UniV3S32},
2233 .OperandMapping: {{UniInVgprV3S32}, {IntrId, VgprV16S16, Vgpr32, Vgpr32}}});
2234
2235 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_sr_pk16_bf6_f32,
2236 amdgcn_cvt_scalef32_sr_pk16_fp6_f32},
2237 FastTypes: Standard)
2238 .Any(Init: {.Predicate: {DivV3S32}, .OperandMapping: {{VgprV3S32}, {IntrId, VgprV16S32, Vgpr32, Vgpr32}}})
2239 .Any(Init: {.Predicate: {UniV3S32},
2240 .OperandMapping: {{UniInVgprV3S32}, {IntrId, VgprV16S32, Vgpr32, Vgpr32}}});
2241
2242 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_sr_pk8_bf8_f16,
2243 amdgcn_cvt_scalef32_sr_pk8_fp8_f16,
2244 amdgcn_cvt_scalef32_sr_pk8_bf8_bf16,
2245 amdgcn_cvt_scalef32_sr_pk8_fp8_bf16},
2246 FastTypes: Standard)
2247 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {IntrId, VgprV8S16, Vgpr32, Vgpr32}}})
2248 .Any(Init: {.Predicate: {UniV2S32},
2249 .OperandMapping: {{UniInVgprV2S32}, {IntrId, VgprV8S16, Vgpr32, Vgpr32}}});
2250
2251 addRulesForIOpcs(
2252 OpcList: {amdgcn_cvt_scalef32_sr_pk8_bf8_f32, amdgcn_cvt_scalef32_sr_pk8_fp8_f32},
2253 FastTypes: Standard)
2254 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {IntrId, VgprV8S32, Vgpr32, Vgpr32}}})
2255 .Any(Init: {.Predicate: {UniV2S32},
2256 .OperandMapping: {{UniInVgprV2S32}, {IntrId, VgprV8S32, Vgpr32, Vgpr32}}});
2257
2258 addRulesForIOpcs(
2259 OpcList: {amdgcn_cvt_scalef32_sr_pk8_fp4_f16, amdgcn_cvt_scalef32_sr_pk8_fp4_bf16},
2260 FastTypes: Standard)
2261 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, VgprV8S16, Vgpr32, Vgpr32}})
2262 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, VgprV8S16, Vgpr32, Vgpr32}});
2263
2264 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_sr_pk8_fp4_f32}, FastTypes: Standard)
2265 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, VgprV8S32, Vgpr32, Vgpr32}})
2266 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, VgprV8S32, Vgpr32, Vgpr32}});
2267
2268 addRulesForIOpcs(
2269 OpcList: {amdgcn_cvt_scale_pk16_f16_bf6, amdgcn_cvt_scale_pk16_f16_fp6,
2270 amdgcn_cvt_scale_pk16_bf16_bf6, amdgcn_cvt_scale_pk16_bf16_fp6},
2271 FastTypes: Standard)
2272 .Any(Init: {.Predicate: {DivV16S16}, .OperandMapping: {{VgprV16S16}, {IntrId, VgprV3S32, Vgpr32}}})
2273 .Any(Init: {.Predicate: {UniV16S16}, .OperandMapping: {{UniInVgprV16S16}, {IntrId, VgprV3S32, Vgpr32}}});
2274
2275 addRulesForIOpcs(
2276 OpcList: {amdgcn_cvt_scale_pk16_f32_bf6, amdgcn_cvt_scale_pk16_f32_fp6}, FastTypes: Standard)
2277 .Any(Init: {.Predicate: {DivV16S32}, .OperandMapping: {{VgprV16S32}, {IntrId, VgprV3S32, Vgpr32}}})
2278 .Any(Init: {.Predicate: {UniV16S32}, .OperandMapping: {{UniInVgprV16S32}, {IntrId, VgprV3S32, Vgpr32}}});
2279
2280 addRulesForIOpcs(
2281 OpcList: {amdgcn_cvt_scale_pk32_f16_bf6, amdgcn_cvt_scale_pk32_f16_fp6,
2282 amdgcn_cvt_scale_pk32_bf16_bf6, amdgcn_cvt_scale_pk32_bf16_fp6},
2283 FastTypes: Standard)
2284 .Any(Init: {.Predicate: {DivV32S16}, .OperandMapping: {{VgprV32S16}, {IntrId, VgprV6S32, Vgpr32}}})
2285 .Any(Init: {.Predicate: {UniV32S16}, .OperandMapping: {{UniInVgprV32S16}, {IntrId, VgprV6S32, Vgpr32}}});
2286
2287 addRulesForIOpcs(
2288 OpcList: {amdgcn_cvt_scale_pk32_f32_bf6, amdgcn_cvt_scale_pk32_f32_fp6}, FastTypes: Standard)
2289 .Any(Init: {.Predicate: {DivV32S32}, .OperandMapping: {{VgprV32S32}, {IntrId, VgprV6S32, Vgpr32}}})
2290 .Any(Init: {.Predicate: {UniV32S32}, .OperandMapping: {{UniInVgprV32S32}, {IntrId, VgprV6S32, Vgpr32}}});
2291
2292 addRulesForIOpcs(OpcList: {amdgcn_cvt_scale_pk8_f16_bf8, amdgcn_cvt_scale_pk8_f16_fp8,
2293 amdgcn_cvt_scale_pk8_bf16_bf8,
2294 amdgcn_cvt_scale_pk8_bf16_fp8},
2295 FastTypes: Standard)
2296 .Any(Init: {.Predicate: {DivV8S16}, .OperandMapping: {{VgprV8S16}, {IntrId, VgprV2S32, Vgpr32}}})
2297 .Any(Init: {.Predicate: {UniV8S16}, .OperandMapping: {{UniInVgprV8S16}, {IntrId, VgprV2S32, Vgpr32}}});
2298
2299 addRulesForIOpcs(
2300 OpcList: {amdgcn_cvt_scale_pk8_f16_fp4, amdgcn_cvt_scale_pk8_bf16_fp4}, FastTypes: Standard)
2301 .Any(Init: {.Predicate: {DivV8S16}, .OperandMapping: {{VgprV8S16}, {IntrId, Vgpr32, Vgpr32}}})
2302 .Any(Init: {.Predicate: {UniV8S16}, .OperandMapping: {{UniInVgprV8S16}, {IntrId, Vgpr32, Vgpr32}}});
2303
2304 addRulesForIOpcs(OpcList: {amdgcn_cvt_scale_pk8_f32_bf8, amdgcn_cvt_scale_pk8_f32_fp8},
2305 FastTypes: Standard)
2306 .Any(Init: {.Predicate: {DivV8S32}, .OperandMapping: {{VgprV8S32}, {IntrId, VgprV2S32, Vgpr32}}})
2307 .Any(Init: {.Predicate: {UniV8S32}, .OperandMapping: {{UniInVgprV8S32}, {IntrId, VgprV2S32, Vgpr32}}});
2308
2309 addRulesForIOpcs(OpcList: {amdgcn_cvt_scale_pk8_f32_fp4}, FastTypes: Standard)
2310 .Any(Init: {.Predicate: {DivV8S32}, .OperandMapping: {{VgprV8S32}, {IntrId, Vgpr32, Vgpr32}}})
2311 .Any(Init: {.Predicate: {UniV8S32}, .OperandMapping: {{UniInVgprV8S32}, {IntrId, Vgpr32, Vgpr32}}});
2312
2313 addRulesForIOpcs(
2314 OpcList: {amdgcn_cvt_scalef32_pk32_bf6_f16, amdgcn_cvt_scalef32_pk32_fp6_f16,
2315 amdgcn_cvt_scalef32_pk32_bf6_bf16, amdgcn_cvt_scalef32_pk32_fp6_bf16},
2316 FastTypes: Standard)
2317 .Any(Init: {.Predicate: {DivV6S32}, .OperandMapping: {{VgprV6S32}, {IntrId, VgprV32S16, Vgpr32}}})
2318 .Any(Init: {.Predicate: {UniV6S32}, .OperandMapping: {{UniInVgprV6S32}, {IntrId, VgprV32S16, Vgpr32}}});
2319
2320 addRulesForIOpcs(
2321 OpcList: {amdgcn_cvt_scalef32_pk32_bf6_f32, amdgcn_cvt_scalef32_pk32_fp6_f32},
2322 FastTypes: Standard)
2323 .Any(Init: {.Predicate: {DivV6S32}, .OperandMapping: {{VgprV6S32}, {IntrId, VgprV32S32, Vgpr32}}})
2324 .Any(Init: {.Predicate: {UniV6S32}, .OperandMapping: {{UniInVgprV6S32}, {IntrId, VgprV32S32, Vgpr32}}});
2325
2326 addRulesForIOpcs(
2327 OpcList: {amdgcn_cvt_scalef32_pk_fp8_f32, amdgcn_cvt_scalef32_pk_bf8_f32},
2328 FastTypes: Standard)
2329 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {IntrId, VgprV2S16, Vgpr32, Vgpr32, Vgpr32}})
2330 .Uni(Ty: V2S16,
2331 RuleApplyIDs: {{UniInVgprV2S16}, {IntrId, VgprV2S16, Vgpr32, Vgpr32, Vgpr32}});
2332
2333 addRulesForIOpcs(
2334 OpcList: {amdgcn_cvt_scalef32_pk_f32_fp8, amdgcn_cvt_scalef32_pk_f32_bf8},
2335 FastTypes: Standard)
2336 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {IntrId, Vgpr32, Vgpr32}}})
2337 .Any(Init: {.Predicate: {UniV2S32}, .OperandMapping: {{UniInVgprV2S32}, {IntrId, Vgpr32, Vgpr32}}});
2338
2339 addRulesForIOpcs(
2340 OpcList: {amdgcn_cvt_scalef32_pk_fp8_f16, amdgcn_cvt_scalef32_pk_bf8_f16,
2341 amdgcn_cvt_scalef32_pk_fp8_bf16, amdgcn_cvt_scalef32_pk_bf8_bf16},
2342 FastTypes: Standard)
2343 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {IntrId, VgprV2S16, VgprV2S16, Vgpr32}})
2344 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {IntrId, VgprV2S16, VgprV2S16, Vgpr32}});
2345
2346 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_pk_f32_fp4}, FastTypes: Standard)
2347 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {IntrId, Vgpr32, Vgpr32}}})
2348 .Any(Init: {.Predicate: {UniV2S32}, .OperandMapping: {{UniInVgprV2S32}, {IntrId, Vgpr32, Vgpr32}}});
2349
2350 addRulesForIOpcs(OpcList: {amdgcn_cvt_scalef32_pk_fp4_f32}, FastTypes: Standard)
2351 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32, Vgpr32}})
2352 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32, Vgpr32}});
2353
2354 addRulesForIOpcs(
2355 OpcList: {amdgcn_cvt_scalef32_pk_f16_fp4, amdgcn_cvt_scalef32_pk_f16_fp8,
2356 amdgcn_cvt_scalef32_pk_f16_bf8, amdgcn_cvt_scalef32_pk_bf16_fp4,
2357 amdgcn_cvt_scalef32_pk_bf16_fp8, amdgcn_cvt_scalef32_pk_bf16_bf8},
2358 FastTypes: Standard)
2359 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {IntrId, Vgpr32, Vgpr32}})
2360 .Uni(Ty: V2S16, RuleApplyIDs: {{UniInVgprV2S16}, {IntrId, Vgpr32, Vgpr32}});
2361
2362 addRulesForIOpcs(
2363 OpcList: {amdgcn_cvt_scalef32_pk32_f32_fp6, amdgcn_cvt_scalef32_pk32_f32_bf6},
2364 FastTypes: Standard)
2365 .Any(Init: {.Predicate: {DivV32S32}, .OperandMapping: {{VgprV32S32}, {IntrId, VgprV6S32, Vgpr32}}})
2366 .Any(Init: {.Predicate: {UniV32S32}, .OperandMapping: {{UniInVgprV32S32}, {IntrId, VgprV6S32, Vgpr32}}});
2367
2368 addRulesForIOpcs(
2369 OpcList: {amdgcn_cvt_scalef32_pk32_f16_fp6, amdgcn_cvt_scalef32_pk32_f16_bf6,
2370 amdgcn_cvt_scalef32_pk32_bf16_fp6, amdgcn_cvt_scalef32_pk32_bf16_bf6},
2371 FastTypes: Standard)
2372 .Any(Init: {.Predicate: {DivV32S16}, .OperandMapping: {{VgprV32S16}, {IntrId, VgprV6S32, Vgpr32}}})
2373 .Any(Init: {.Predicate: {UniV32S16}, .OperandMapping: {{UniInVgprV32S16}, {IntrId, VgprV6S32, Vgpr32}}});
2374
2375 addRulesForIOpcs(
2376 OpcList: {amdgcn_cvt_scalef32_pk_fp4_f16, amdgcn_cvt_scalef32_pk_fp4_bf16},
2377 FastTypes: Standard)
2378 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, VgprV2S16, Vgpr32}})
2379 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, VgprV2S16, Vgpr32}});
2380
2381 addRulesForIOpcs(OpcList: {amdgcn_global_load_tr_b64})
2382 .Any(Init: {.Predicate: {DivB64, _, UniP1}, .OperandMapping: {{VgprB64}, {IntrId, SgprP1}}})
2383 .Any(Init: {.Predicate: {DivB64, _, DivP1}, .OperandMapping: {{VgprB64}, {IntrId, VgprP1}}})
2384 .Any(Init: {.Predicate: {DivB32, _, UniP1}, .OperandMapping: {{VgprB32}, {IntrId, SgprP1}}})
2385 .Any(Init: {.Predicate: {DivB32, _, DivP1}, .OperandMapping: {{VgprB32}, {IntrId, VgprP1}}});
2386
2387 addRulesForIOpcs(OpcList: {amdgcn_global_load_tr_b128})
2388 .Any(Init: {.Predicate: {DivB64, _, UniP1}, .OperandMapping: {{VgprB64}, {IntrId, SgprP1}}})
2389 .Any(Init: {.Predicate: {DivB64, _, DivP1}, .OperandMapping: {{VgprB64}, {IntrId, VgprP1}}})
2390 .Any(Init: {.Predicate: {DivB128, _, UniP1}, .OperandMapping: {{VgprB128}, {IntrId, SgprP1}}})
2391 .Any(Init: {.Predicate: {DivB128, _, DivP1}, .OperandMapping: {{VgprB128}, {IntrId, VgprP1}}});
2392
2393 addRulesForIOpcs(OpcList: {amdgcn_global_load_tr4_b64})
2394 .Any(Init: {.Predicate: {DivV2S32, _, UniP1}, .OperandMapping: {{VgprV2S32}, {IntrId, SgprP1}}})
2395 .Any(Init: {.Predicate: {DivV2S32, _, DivP1}, .OperandMapping: {{VgprV2S32}, {IntrId, VgprP1}}});
2396
2397 addRulesForIOpcs(OpcList: {amdgcn_global_load_tr6_b96})
2398 .Any(Init: {.Predicate: {DivV3S32, _, UniP1}, .OperandMapping: {{VgprV3S32}, {IntrId, SgprP1}}})
2399 .Any(Init: {.Predicate: {DivV3S32, _, DivP1}, .OperandMapping: {{VgprV3S32}, {IntrId, VgprP1}}});
2400
2401 addRulesForIOpcs(OpcList: {amdgcn_ds_load_tr4_b64, amdgcn_ds_load_tr8_b64})
2402 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {IntrId, VgprP3}}});
2403
2404 addRulesForIOpcs(OpcList: {amdgcn_ds_load_tr6_b96})
2405 .Any(Init: {.Predicate: {DivV3S32}, .OperandMapping: {{VgprV3S32}, {IntrId, VgprP3}}});
2406
2407 addRulesForIOpcs(OpcList: {amdgcn_ds_load_tr16_b128})
2408 .Any(Init: {.Predicate: {DivB128}, .OperandMapping: {{VgprB128}, {IntrId, VgprP3}}});
2409
2410 addRulesForIOpcs(OpcList: {amdgcn_global_atomic_ordered_add_b64})
2411 .Any(Init: {.Predicate: {DivS64}, .OperandMapping: {{Vgpr64}, {IntrId, VgprP1, Vgpr64}}});
2412
2413 addRulesForIOpcs(
2414 OpcList: {amdgcn_global_atomic_fmin_num, amdgcn_global_atomic_fmax_num}, FastTypes: Standard)
2415 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, VgprP1, Vgpr32}});
2416
2417 addRulesForIOpcs(OpcList: {amdgcn_flat_atomic_fmin_num, amdgcn_flat_atomic_fmax_num},
2418 FastTypes: Standard)
2419 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, VgprP0, Vgpr32}});
2420
2421 addRulesForIOpcs(OpcList: {amdgcn_raw_buffer_load_lds})
2422 .Any(Init: {.Predicate: {_}, .OperandMapping: {{}, {IntrId, SgprV4S32, SgprP3, Imm, Vgpr32, Sgpr32}}});
2423
2424 addRulesForIOpcs(OpcList: {amdgcn_raw_buffer_load_async_lds})
2425 .Any(Init: {.Predicate: {_}, .OperandMapping: {{}, {IntrId, SgprV4S32, SgprB32_M0, Imm, Vgpr32, Sgpr32}}});
2426
2427 addRulesForIOpcs(OpcList: {amdgcn_struct_buffer_load_async_lds})
2428 .Any(
2429 Init: {.Predicate: {_},
2430 .OperandMapping: {{}, {IntrId, SgprV4S32, SgprB32_M0, Imm, Vgpr32, Vgpr32, Sgpr32}}});
2431
2432 addRulesForIOpcs(OpcList: {amdgcn_struct_buffer_load_lds})
2433 .Any(Init: {.Predicate: {_},
2434 .OperandMapping: {{}, {IntrId, SgprV4S32, SgprP3, Imm, Vgpr32, Vgpr32, Sgpr32}}});
2435
2436 addRulesForIOpcs(OpcList: {amdgcn_raw_ptr_buffer_load_lds})
2437 .Any(Init: {.Predicate: {_}, .OperandMapping: {{}, {IntrId, SgprP8, SgprP3, Imm, Vgpr32, Sgpr32}}});
2438
2439 addRulesForIOpcs(OpcList: {amdgcn_raw_ptr_buffer_load_async_lds})
2440 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, SgprP8, SgprB32_M0, Imm, VgprB32, SgprB32}}});
2441
2442 addRulesForIOpcs(OpcList: {amdgcn_struct_ptr_buffer_load_async_lds})
2443 .Any(Init: {.Predicate: {_},
2444 .OperandMapping: {{}, {IntrId, SgprP8, SgprB32_M0, Imm, Vgpr32, Vgpr32, Sgpr32}}});
2445
2446 addRulesForIOpcs(OpcList: {amdgcn_struct_ptr_buffer_load_lds})
2447 .Any(Init: {.Predicate: {_}, .OperandMapping: {{}, {IntrId, SgprP8, SgprP3, Imm, Vgpr32, Vgpr32, Sgpr32}}});
2448
2449 addRulesForIOpcs(
2450 OpcList: {amdgcn_global_load_lds, amdgcn_load_to_lds, amdgcn_load_async_to_lds})
2451 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, VgprP1, SgprB32_M0}}});
2452
2453 addRulesForIOpcs(OpcList: {amdgcn_global_load_async_to_lds_b8,
2454 amdgcn_global_load_async_to_lds_b32,
2455 amdgcn_global_load_async_to_lds_b64,
2456 amdgcn_global_load_async_to_lds_b128,
2457 amdgcn_global_store_async_from_lds_b8,
2458 amdgcn_global_store_async_from_lds_b32,
2459 amdgcn_global_store_async_from_lds_b64,
2460 amdgcn_global_store_async_from_lds_b128})
2461 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, VgprP1, VgprP3}}});
2462
2463 addRulesForIOpcs(OpcList: {amdgcn_global_load_async_lds})
2464 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, VgprP1, SgprB32_M0}}});
2465
2466 addRulesForIOpcs(OpcList: {amdgcn_tensor_load_to_lds, amdgcn_tensor_store_from_lds})
2467 .Any(Init: {.Predicate: {},
2468 .OperandMapping: {{},
2469 {IntrId, SgprV4S32_ReadFirstLane, SgprV8S32_ReadFirstLane,
2470 SgprV4S32_ReadFirstLane, SgprV4S32_ReadFirstLane,
2471 SgprV8S32_ReadFirstLane}}});
2472
2473 addRulesForIOpcs(OpcList: {amdgcn_cluster_load_b32})
2474 .Any(Init: {.Predicate: {UniB32}, .OperandMapping: {{UniInVgprB32}, {IntrId, SgprP1, Imm, SgprB32_M0}}})
2475 .Any(Init: {.Predicate: {DivB32, _, UniP1}, .OperandMapping: {{VgprB32}, {IntrId, SgprP1, Imm, SgprB32_M0}}})
2476 .Any(
2477 Init: {.Predicate: {DivB32, _, DivP1}, .OperandMapping: {{VgprB32}, {IntrId, VgprP1, Imm, SgprB32_M0}}});
2478
2479 addRulesForIOpcs(OpcList: {amdgcn_cluster_load_b64})
2480 .Any(Init: {.Predicate: {UniB64}, .OperandMapping: {{UniInVgprB64}, {IntrId, SgprP1, Imm, SgprB32_M0}}})
2481 .Any(Init: {.Predicate: {DivB64, _, UniP1}, .OperandMapping: {{VgprB64}, {IntrId, SgprP1, Imm, SgprB32_M0}}})
2482 .Any(
2483 Init: {.Predicate: {DivB64, _, DivP1}, .OperandMapping: {{VgprB64}, {IntrId, VgprP1, Imm, SgprB32_M0}}});
2484
2485 addRulesForIOpcs(OpcList: {amdgcn_cluster_load_b128})
2486 .Any(Init: {.Predicate: {UniB128}, .OperandMapping: {{UniInVgprB128}, {IntrId, SgprP1, Imm, SgprB32_M0}}})
2487 .Any(Init: {.Predicate: {DivB128, _, UniP1},
2488 .OperandMapping: {{VgprB128}, {IntrId, SgprP1, Imm, SgprB32_M0}}})
2489 .Any(Init: {.Predicate: {DivB128, _, DivP1},
2490 .OperandMapping: {{VgprB128}, {IntrId, VgprP1, Imm, SgprB32_M0}}});
2491
2492 addRulesForIOpcs(OpcList: {amdgcn_cluster_load_async_to_lds_b8,
2493 amdgcn_cluster_load_async_to_lds_b32,
2494 amdgcn_cluster_load_async_to_lds_b64,
2495 amdgcn_cluster_load_async_to_lds_b128})
2496 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, VgprP1, VgprP3, Imm, Imm, SgprB32_M0}}});
2497
2498 addRulesForIOpcs(OpcList: {amdgcn_perm_pk16_b4_u4}, FastTypes: StandardB)
2499 .Uni(Ty: B64, RuleApplyIDs: {{UniInVgprB64}, {IntrId, Vgpr32, Vgpr32, VgprV2S32}})
2500 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {IntrId, Vgpr32, Vgpr32, VgprV2S32}});
2501
2502 addRulesForIOpcs(OpcList: {amdgcn_perm_pk16_b6_u4}, FastTypes: StandardB)
2503 .Uni(Ty: B96, RuleApplyIDs: {{UniInVgprB96}, {IntrId, Vgpr32, VgprB64, VgprV2S32}})
2504 .Div(Ty: B96, RuleApplyIDs: {{VgprB96}, {IntrId, Vgpr32, VgprB64, VgprV2S32}});
2505
2506 addRulesForIOpcs(OpcList: {amdgcn_perm_pk16_b8_u4}, FastTypes: StandardB)
2507 .Uni(Ty: B128, RuleApplyIDs: {{UniInVgprB128}, {IntrId, VgprB64, VgprB64, VgprV2S32}})
2508 .Div(Ty: B128, RuleApplyIDs: {{VgprB128}, {IntrId, VgprB64, VgprB64, VgprV2S32}});
2509
2510 addRulesForIOpcs(OpcList: {amdgcn_wwm, amdgcn_strict_wwm, amdgcn_wqm, amdgcn_softwqm,
2511 amdgcn_strict_wqm},
2512 FastTypes: StandardB)
2513 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {IntrId, VgprB32}})
2514 .Uni(Ty: B32, RuleApplyIDs: {{SgprB32}, {IntrId, SgprB32}})
2515 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {IntrId, VgprB64}})
2516 .Uni(Ty: B64, RuleApplyIDs: {{SgprB64}, {IntrId, SgprB64}})
2517 .Div(Ty: B96, RuleApplyIDs: {{VgprB96}, {IntrId, VgprB96}})
2518 .Uni(Ty: B96, RuleApplyIDs: {{SgprB96}, {IntrId, SgprB96}})
2519 .Div(Ty: B128, RuleApplyIDs: {{VgprB128}, {IntrId, VgprB128}})
2520 .Uni(Ty: B128, RuleApplyIDs: {{SgprB128}, {IntrId, SgprB128}})
2521 .Any(Init: {.Predicate: {UniB256}, .OperandMapping: {{SgprB256}, {IntrId, SgprB256}}})
2522 .Any(Init: {.Predicate: {DivB256}, .OperandMapping: {{VgprB256}, {IntrId, VgprB256}}})
2523 .Any(Init: {.Predicate: {UniB512}, .OperandMapping: {{SgprB512}, {IntrId, SgprB512}}})
2524 .Any(Init: {.Predicate: {DivB512}, .OperandMapping: {{VgprB512}, {IntrId, VgprB512}}});
2525
2526 addRulesForIOpcs(OpcList: {amdgcn_init_whole_wave}).Any(Init: {.Predicate: {DivS1}, .OperandMapping: {{Vcc}, {IntrId}}});
2527
2528 addRulesForIOpcs(OpcList: {amdgcn_kill, amdgcn_wqm_demote})
2529 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, Vcc}}});
2530
2531 addRulesForIOpcs(OpcList: {amdgcn_set_inactive}, FastTypes: StandardB)
2532 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {IntrId, VgprB32, VgprB32}});
2533
2534 addRulesForIOpcs(OpcList: {amdgcn_set_inactive_chain_arg}, FastTypes: Standard)
2535 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32}});
2536
2537 addRulesForIOpcs(OpcList: {amdgcn_cvt_sr_bf16_f32, amdgcn_cvt_sr_f16_f32}, FastTypes: Standard)
2538 .Div(Ty: V2S16, RuleApplyIDs: {{VgprV2S16}, {IntrId, VgprV2S16, Vgpr32, Vgpr32}});
2539
2540 addRulesForIOpcs(OpcList: {amdgcn_ballot}, FastTypes: Standard)
2541 .Uni(Ty: S64, RuleApplyIDs: {{Sgpr64}, {IntrId, Vcc}})
2542 .Uni(Ty: S32, RuleApplyIDs: {{Sgpr32}, {IntrId, Vcc}});
2543
2544 addRulesForIOpcs(OpcList: {amdgcn_inverse_ballot})
2545 .Any(Init: {.Predicate: {DivS1, _, S32}, .OperandMapping: {{Vcc}, {IntrId, SgprB32_ReadFirstLane}}})
2546 .Any(Init: {.Predicate: {DivS1, _, S64}, .OperandMapping: {{Vcc}, {IntrId, SgprB64_ReadFirstLane}}});
2547
2548 addRulesForIOpcs(OpcList: {amdgcn_live_mask, amdgcn_ps_live})
2549 .Any(Init: {.Predicate: {DivS1}, .OperandMapping: {{Vcc}, {}}});
2550
2551 addRulesForIOpcs(OpcList: {amdgcn_mov_dpp, amdgcn_mov_dpp8}, FastTypes: StandardB)
2552 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {IntrId, VgprB32}})
2553 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {IntrId, VgprB64}});
2554
2555 addRulesForIOpcs(OpcList: {amdgcn_update_dpp}, FastTypes: StandardB)
2556 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {IntrId, VgprB32, VgprB32}})
2557 .Div(Ty: B64, RuleApplyIDs: {{VgprB64}, {IntrId, VgprB64, VgprB64}});
2558
2559 addRulesForIOpcs(OpcList: {amdgcn_sin, amdgcn_cos, amdgcn_tanh}, FastTypes: Standard)
2560 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, Vgpr16}})
2561 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, Vgpr16}})
2562 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32}})
2563 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32}});
2564
2565 addRulesForIOpcs(OpcList: {amdgcn_trig_preop}, FastTypes: Standard)
2566 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {IntrId, Vgpr64, Vgpr32}})
2567 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {IntrId, Vgpr64, Vgpr32}});
2568
2569 addRulesForIOpcs(OpcList: {amdgcn_exp2})
2570 .Any(Init: {.Predicate: {DivS16}, .OperandMapping: {{Vgpr16}, {IntrId, Vgpr16}}})
2571 .Any(Init: {.Predicate: {UniBF16}, .OperandMapping: {{UniInVgprS16}, {IntrId, Vgpr16}}})
2572 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{Sgpr16}, {IntrId, Sgpr16}}}, STPred: hasPST)
2573 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{UniInVgprS16}, {IntrId, Vgpr16}}}, STPred: !hasPST)
2574 .Any(Init: {.Predicate: {DivS32}, .OperandMapping: {{Vgpr32}, {IntrId, Vgpr32}}})
2575 .Any(Init: {.Predicate: {UniS32}, .OperandMapping: {{Sgpr32}, {IntrId, Sgpr32}}}, STPred: hasPST)
2576 .Any(Init: {.Predicate: {UniS32}, .OperandMapping: {{UniInVgprS32}, {IntrId, Vgpr32}}}, STPred: !hasPST);
2577
2578 addRulesForIOpcs(OpcList: {amdgcn_rcp, amdgcn_sqrt})
2579 .Any(Init: {.Predicate: {DivS16}, .OperandMapping: {{Vgpr16}, {IntrId, Vgpr16}}})
2580 .Any(Init: {.Predicate: {UniBF16}, .OperandMapping: {{UniInVgprS16}, {IntrId, Vgpr16}}})
2581 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{Sgpr16}, {IntrId, Sgpr16}}}, STPred: hasPST)
2582 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{UniInVgprS16}, {IntrId, Vgpr16}}}, STPred: !hasPST)
2583 .Any(Init: {.Predicate: {DivS32}, .OperandMapping: {{Vgpr32}, {IntrId, Vgpr32}}})
2584 .Any(Init: {.Predicate: {UniS32}, .OperandMapping: {{Sgpr32}, {IntrId, Sgpr32}}}, STPred: hasPST)
2585 .Any(Init: {.Predicate: {UniS32}, .OperandMapping: {{UniInVgprS32}, {IntrId, Vgpr32}}}, STPred: !hasPST)
2586 .Any(Init: {.Predicate: {DivS64}, .OperandMapping: {{Vgpr64}, {IntrId, Vgpr64}}})
2587 .Any(Init: {.Predicate: {UniS64}, .OperandMapping: {{UniInVgprS64}, {IntrId, Vgpr64}}});
2588
2589 addRulesForIOpcs(OpcList: {amdgcn_log})
2590 .Any(Init: {.Predicate: {DivS16}, .OperandMapping: {{Vgpr16}, {IntrId, Vgpr16}}})
2591 .Any(Init: {.Predicate: {UniBF16}, .OperandMapping: {{UniInVgprS16}, {IntrId, Vgpr16}}})
2592 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{Sgpr16}, {IntrId, Sgpr16}}}, STPred: hasPST)
2593 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{UniInVgprS16}, {IntrId, Vgpr16}}}, STPred: !hasPST)
2594 .Any(Init: {.Predicate: {DivS32}, .OperandMapping: {{Vgpr32}, {IntrId, Vgpr32}}})
2595 .Any(Init: {.Predicate: {UniS32}, .OperandMapping: {{Sgpr32}, {IntrId, Sgpr32}}}, STPred: hasPST)
2596 .Any(Init: {.Predicate: {UniS32}, .OperandMapping: {{UniInVgprS32}, {IntrId, Vgpr32}}}, STPred: !hasPST);
2597
2598 addRulesForIOpcs(OpcList: {amdgcn_ds_atomic_async_barrier_arrive_b64})
2599 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, VgprP3}}});
2600
2601 addRulesForIOpcs(OpcList: {amdgcn_ds_atomic_barrier_arrive_rtn_b64}, FastTypes: Standard)
2602 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {IntrId, VgprP3, Vgpr64}});
2603
2604 addRulesForIOpcs(OpcList: {amdgcn_ds_add_gs_reg_rtn, amdgcn_ds_sub_gs_reg_rtn},
2605 FastTypes: Standard)
2606 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32}})
2607 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {IntrId, Vgpr32}});
2608
2609 addRulesForIOpcs(OpcList: {amdgcn_ds_append, amdgcn_ds_consume}, FastTypes: Standard)
2610 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, SgprB32_M0}})
2611 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, SgprB32_M0}});
2612
2613 addRulesForIOpcs(
2614 OpcList: {amdgcn_ds_bvh_stack_rtn, amdgcn_ds_bvh_stack_push4_pop1_rtn}, FastTypes: Standard)
2615 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32, Vgpr32}, {IntrId, Vgpr32, Vgpr32, VgprV4S32}});
2616
2617 addRulesForIOpcs(OpcList: {amdgcn_ds_bvh_stack_push8_pop1_rtn}, FastTypes: Standard)
2618 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32, Vgpr32}, {IntrId, Vgpr32, Vgpr32, VgprV8S32}});
2619
2620 addRulesForIOpcs(OpcList: {amdgcn_ds_bvh_stack_push8_pop2_rtn}, FastTypes: Standard)
2621 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64, Vgpr32}, {IntrId, Vgpr32, Vgpr32, VgprV8S32}});
2622
2623 addRulesForIOpcs(OpcList: {amdgcn_ds_gws_sema_p, amdgcn_ds_gws_sema_v,
2624 amdgcn_ds_gws_sema_release_all})
2625 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, SgprB32_M0}}});
2626
2627 addRulesForIOpcs(
2628 OpcList: {amdgcn_ds_gws_barrier, amdgcn_ds_gws_init, amdgcn_ds_gws_sema_br})
2629 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {IntrId, Vgpr32, SgprB32_M0}}});
2630
2631 addRulesForIOpcs(OpcList: {amdgcn_ds_ordered_add, amdgcn_ds_ordered_swap}, FastTypes: Standard)
2632 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, SgprB32_M0, Vgpr32}});
2633
2634 addRulesForIOpcs(OpcList: {amdgcn_ds_swizzle}, FastTypes: Standard)
2635 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32}})
2636 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32}});
2637
2638 addRulesForIOpcs(OpcList: {amdgcn_permlane16_var, amdgcn_permlanex16_var}, FastTypes: Standard)
2639 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
2640
2641 addRulesForIOpcs(OpcList: {amdgcn_permlane16_swap, amdgcn_permlane32_swap}, FastTypes: Standard)
2642 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32, Vgpr32}, {IntrId, Vgpr32, Vgpr32}});
2643
2644 addRulesForIOpcs(OpcList: {amdgcn_permlane64}, FastTypes: StandardB)
2645 .Div(Ty: B32, RuleApplyIDs: {{VgprB32}, {IntrId, VgprB32}});
2646
2647 addRulesForIOpcs(OpcList: {amdgcn_ds_read_tr4_b64, amdgcn_ds_read_tr8_b64})
2648 .Any(Init: {.Predicate: {DivV2S32}, .OperandMapping: {{VgprV2S32}, {IntrId, VgprP3}}});
2649
2650 addRulesForIOpcs(OpcList: {amdgcn_ds_read_tr6_b96})
2651 .Any(Init: {.Predicate: {DivV3S32}, .OperandMapping: {{VgprV3S32}, {IntrId, VgprP3}}});
2652
2653 addRulesForIOpcs(OpcList: {amdgcn_ds_read_tr16_b64})
2654 .Any(Init: {.Predicate: {DivV4S16}, .OperandMapping: {{VgprV4S16}, {IntrId, VgprP3}}});
2655
2656 addRulesForIOpcs(OpcList: {amdgcn_interp_p1}, FastTypes: Standard)
2657 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Imm, Imm, SgprB32_M0}});
2658
2659 addRulesForIOpcs(OpcList: {amdgcn_interp_p1_f16}, FastTypes: Standard)
2660 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Imm, Imm, Imm, SgprB32_M0}});
2661
2662 addRulesForIOpcs(OpcList: {amdgcn_interp_p2}, FastTypes: Standard)
2663 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Imm, Imm, SgprB32_M0}});
2664
2665 addRulesForIOpcs(OpcList: {amdgcn_interp_p2_f16}, FastTypes: Standard)
2666 .Div(Ty: S16,
2667 RuleApplyIDs: {{Vgpr16}, {IntrId, Vgpr32, Vgpr32, Imm, Imm, Imm, SgprB32_M0}});
2668
2669 addRulesForIOpcs(OpcList: {amdgcn_interp_mov}, FastTypes: Standard)
2670 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Imm, Imm, Imm, SgprB32_M0}});
2671
2672 addRulesForIOpcs(OpcList: {amdgcn_interp_inreg_p10, amdgcn_interp_inreg_p2,
2673 amdgcn_interp_inreg_p10_f16, amdgcn_interp_p10_rtz_f16},
2674 FastTypes: Standard)
2675 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
2676 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
2677
2678 addRulesForIOpcs(OpcList: {amdgcn_interp_inreg_p2_f16, amdgcn_interp_p2_rtz_f16},
2679 FastTypes: Standard)
2680 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
2681 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, Vgpr32, Vgpr32, Vgpr32}});
2682
2683 addRulesForIOpcs(OpcList: {amdgcn_frexp_exp})
2684 .Any(Init: {.Predicate: {UniS16}, .OperandMapping: {{UniInVgprS16}, {IntrId, Vgpr16}}})
2685 .Any(Init: {.Predicate: {DivS16}, .OperandMapping: {{Vgpr16}, {IntrId, Vgpr16}}})
2686 .Any(Init: {.Predicate: {UniS32, _, S32}, .OperandMapping: {{UniInVgprS32}, {IntrId, Vgpr32}}})
2687 .Any(Init: {.Predicate: {DivS32, _, S32}, .OperandMapping: {{Vgpr32}, {IntrId, Vgpr32}}})
2688 .Any(Init: {.Predicate: {UniS32, _, S64}, .OperandMapping: {{UniInVgprS32}, {IntrId, Vgpr64}}})
2689 .Any(Init: {.Predicate: {DivS32, _, S64}, .OperandMapping: {{Vgpr32}, {IntrId, Vgpr64}}});
2690
2691 addRulesForIOpcs(OpcList: {amdgcn_div_fmas}, FastTypes: Standard)
2692 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32, Vcc}})
2693 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32, Vcc}})
2694 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {IntrId, Vgpr64, Vgpr64, Vgpr64, Vcc}})
2695 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {IntrId, Vgpr64, Vgpr64, Vgpr64, Vcc}});
2696
2697 addRulesForIOpcs(OpcList: {amdgcn_div_fixup}, FastTypes: Standard)
2698 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, Vgpr16, Vgpr16, Vgpr16}})
2699 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, Vgpr16, Vgpr16, Vgpr16}})
2700 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
2701 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Vgpr32, Vgpr32, Vgpr32}})
2702 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64}, {IntrId, Vgpr64, Vgpr64, Vgpr64}})
2703 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64}, {IntrId, Vgpr64, Vgpr64, Vgpr64}});
2704
2705 addRulesForIOpcs(OpcList: {amdgcn_div_scale}, FastTypes: Standard)
2706 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32, Vcc}, {IntrId, Vgpr32, Vgpr32}})
2707 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32, UniInVcc}, {IntrId, Vgpr32, Vgpr32}})
2708 .Div(Ty: S64, RuleApplyIDs: {{Vgpr64, Vcc}, {IntrId, Vgpr64, Vgpr64}})
2709 .Uni(Ty: S64, RuleApplyIDs: {{UniInVgprS64, UniInVcc}, {IntrId, Vgpr64, Vgpr64}});
2710
2711 addRulesForIOpcs(
2712 OpcList: {amdgcn_fdot2, amdgcn_fdot2_f32_bf16, amdgcn_sdot2, amdgcn_udot2},
2713 FastTypes: Standard)
2714 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, VgprV2S16, VgprV2S16, Vgpr32}})
2715 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, VgprV2S16, VgprV2S16, Vgpr32}});
2716
2717 addRulesForIOpcs(OpcList: {amdgcn_fdot2_f16_f16, amdgcn_fdot2_bf16_bf16}, FastTypes: Standard)
2718 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, VgprV2S16, VgprV2S16, Vgpr16}})
2719 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, VgprV2S16, VgprV2S16, Vgpr16}});
2720
2721 addRulesForIOpcs(OpcList: {amdgcn_sudot4, amdgcn_sudot8}, FastTypes: Standard)
2722 .Uni(Ty: S32, RuleApplyIDs: {{UniInVgprS32}, {IntrId, Imm, Vgpr32, Imm, Vgpr32, Vgpr32}})
2723 .Div(Ty: S32, RuleApplyIDs: {{Vgpr32}, {IntrId, Imm, Vgpr32, Imm, Vgpr32, Vgpr32}});
2724
2725 addRulesForIOpcs(OpcList: {amdgcn_s_alloc_vgpr})
2726 .Any(Init: {.Predicate: {UniS1}, .OperandMapping: {{Sgpr32Trunc}, {IntrId, SgprB32_ReadFirstLane}}});
2727
2728 addRulesForIOpcs(OpcList: {amdgcn_sat_pk4_i4_i8, amdgcn_sat_pk4_u4_u8}, FastTypes: Standard)
2729 .Uni(Ty: S16, RuleApplyIDs: {{UniInVgprS16}, {IntrId, Vgpr32}})
2730 .Div(Ty: S16, RuleApplyIDs: {{Vgpr16}, {IntrId, Vgpr32}});
2731
2732 bool HasGFX90AInsts = ST->hasGFX90AInsts();
2733
2734 // On gfx90a+ both AGPR-form and VGPR-form exists
2735 addRulesForIOpcs(OpcList: {amdgcn_mfma_f32_32x32x1f32, amdgcn_mfma_f32_16x16x1f32,
2736 amdgcn_mfma_f32_4x4x1f32, amdgcn_mfma_f32_32x32x2f32,
2737 amdgcn_mfma_f32_16x16x4f32, amdgcn_mfma_f32_32x32x4f16,
2738 amdgcn_mfma_f32_16x16x4f16, amdgcn_mfma_f32_4x4x4f16,
2739 amdgcn_mfma_f32_32x32x8f16, amdgcn_mfma_f32_16x16x16f16,
2740 amdgcn_mfma_i32_32x32x4i8, amdgcn_mfma_i32_16x16x4i8,
2741 amdgcn_mfma_i32_4x4x4i8, amdgcn_mfma_i32_32x32x8i8,
2742 amdgcn_mfma_i32_16x16x16i8, amdgcn_mfma_f32_32x32x2bf16,
2743 amdgcn_mfma_f32_16x16x2bf16, amdgcn_mfma_f32_4x4x2bf16,
2744 amdgcn_mfma_f32_32x32x4bf16, amdgcn_mfma_f32_16x16x8bf16})
2745 .Any(Init: {.Predicate: {DivAnyTy},
2746 .OperandMapping: {{AgprAnyTy}, {IntrId, VgprAnyTy, VgprAnyTy, AgprAnyTy}}},
2747 STPred: !HasGFX90AInsts)
2748 .Any(Init: {.Predicate: {DivAnyTy},
2749 .OperandMapping: {{VgprOrAgprAnyTy},
2750 {IntrId, VgprAnyTy, VgprAnyTy, VgprOrAgprAnyTy}}},
2751 STPred: HasGFX90AInsts);
2752
2753 // gfx90a+ only MFMAs
2754 addRulesForIOpcs(OpcList: {
2755 amdgcn_mfma_f32_32x32x4bf16_1k,
2756 amdgcn_mfma_f32_16x16x4bf16_1k,
2757 amdgcn_mfma_f32_4x4x4bf16_1k,
2758 amdgcn_mfma_f32_32x32x8bf16_1k,
2759 amdgcn_mfma_f32_16x16x16bf16_1k,
2760 amdgcn_mfma_f64_16x16x4f64,
2761 amdgcn_mfma_f64_4x4x4f64,
2762 amdgcn_mfma_i32_16x16x32_i8,
2763 amdgcn_mfma_i32_32x32x16_i8,
2764 amdgcn_mfma_f32_16x16x8_xf32,
2765 amdgcn_mfma_f32_32x32x4_xf32,
2766 amdgcn_mfma_f32_16x16x32_bf8_bf8,
2767 amdgcn_mfma_f32_16x16x32_bf8_fp8,
2768 amdgcn_mfma_f32_16x16x32_fp8_bf8,
2769 amdgcn_mfma_f32_16x16x32_fp8_fp8,
2770 amdgcn_mfma_f32_32x32x16_bf8_bf8,
2771 amdgcn_mfma_f32_32x32x16_bf8_fp8,
2772 amdgcn_mfma_f32_32x32x16_fp8_bf8,
2773 amdgcn_mfma_f32_32x32x16_fp8_fp8,
2774 // gfx950
2775 amdgcn_mfma_f32_16x16x32_f16,
2776 amdgcn_mfma_f32_32x32x16_f16,
2777 amdgcn_mfma_i32_16x16x64_i8,
2778 amdgcn_mfma_i32_32x32x32_i8,
2779 amdgcn_mfma_f32_16x16x32_bf16,
2780 amdgcn_mfma_f32_32x32x16_bf16,
2781 })
2782 .Any(Init: {.Predicate: {DivAnyTy},
2783 .OperandMapping: {{VgprOrAgprAnyTy},
2784 {IntrId, VgprAnyTy, VgprAnyTy, VgprOrAgprAnyTy}}});
2785
2786 addRulesForIOpcs(
2787 OpcList: {// gfx942+
2788 amdgcn_smfmac_f32_16x16x32_f16, amdgcn_smfmac_f32_32x32x16_f16,
2789 amdgcn_smfmac_f32_16x16x32_bf16, amdgcn_smfmac_f32_32x32x16_bf16,
2790 amdgcn_smfmac_i32_16x16x64_i8, amdgcn_smfmac_i32_32x32x32_i8,
2791 amdgcn_smfmac_f32_16x16x64_bf8_bf8, amdgcn_smfmac_f32_16x16x64_bf8_fp8,
2792 amdgcn_smfmac_f32_16x16x64_fp8_bf8, amdgcn_smfmac_f32_16x16x64_fp8_fp8,
2793 amdgcn_smfmac_f32_32x32x32_bf8_bf8, amdgcn_smfmac_f32_32x32x32_bf8_fp8,
2794 amdgcn_smfmac_f32_32x32x32_fp8_bf8, amdgcn_smfmac_f32_32x32x32_fp8_fp8,
2795 // gfx950+
2796 amdgcn_smfmac_f32_16x16x64_f16, amdgcn_smfmac_f32_32x32x32_f16,
2797 amdgcn_smfmac_f32_16x16x64_bf16, amdgcn_smfmac_f32_32x32x32_bf16,
2798 amdgcn_smfmac_i32_16x16x128_i8, amdgcn_smfmac_i32_32x32x64_i8,
2799 amdgcn_smfmac_f32_16x16x128_bf8_bf8, amdgcn_smfmac_f32_16x16x128_bf8_fp8,
2800 amdgcn_smfmac_f32_16x16x128_fp8_bf8, amdgcn_smfmac_f32_16x16x128_fp8_fp8,
2801 amdgcn_smfmac_f32_32x32x64_bf8_bf8, amdgcn_smfmac_f32_32x32x64_bf8_fp8,
2802 amdgcn_smfmac_f32_32x32x64_fp8_bf8, amdgcn_smfmac_f32_32x32x64_fp8_fp8})
2803 .Any(Init: {.Predicate: {DivAnyTy},
2804 .OperandMapping: {{VgprOrAgprAnyTy},
2805 {IntrId, VgprAnyTy, VgprAnyTy, VgprOrAgprAnyTy, VgprAnyTy}}});
2806
2807 addRulesForIOpcs(OpcList: {amdgcn_mfma_scale_f32_32x32x64_f8f6f4,
2808 amdgcn_mfma_scale_f32_16x16x128_f8f6f4})
2809 .Any(Init: {.Predicate: {DivAnyTy},
2810 .OperandMapping: {{VgprOrAgprAnyTy},
2811 {IntrId, VgprAnyTy, VgprAnyTy, VgprOrAgprAnyTy, Imm, Imm, Imm,
2812 Vgpr32, Imm, Vgpr32}}});
2813
2814 // WMMA/SWMMAC intrinsics: all register operands map to VGPR.
2815 addRulesForIOpcs(
2816 OpcList: {// WMMA GFX11+
2817 amdgcn_wmma_f32_16x16x16_f16, amdgcn_wmma_f32_16x16x16_bf16,
2818 amdgcn_wmma_f16_16x16x16_f16, amdgcn_wmma_bf16_16x16x16_bf16,
2819 amdgcn_wmma_f16_16x16x16_f16_tied, amdgcn_wmma_bf16_16x16x16_bf16_tied,
2820 amdgcn_wmma_i32_16x16x16_iu8, amdgcn_wmma_i32_16x16x16_iu4,
2821 // WMMA GFX12
2822 amdgcn_wmma_f32_16x16x16_fp8_fp8, amdgcn_wmma_f32_16x16x16_fp8_bf8,
2823 amdgcn_wmma_f32_16x16x16_bf8_fp8, amdgcn_wmma_f32_16x16x16_bf8_bf8,
2824 amdgcn_wmma_i32_16x16x32_iu4,
2825 // WMMA GFX1250
2826 amdgcn_wmma_f32_16x16x4_f32, amdgcn_wmma_f32_16x16x32_bf16,
2827 amdgcn_wmma_f32_16x16x32_f16, amdgcn_wmma_f16_16x16x32_f16,
2828 amdgcn_wmma_bf16_16x16x32_bf16, amdgcn_wmma_bf16f32_16x16x32_bf16,
2829 amdgcn_wmma_f32_16x16x64_fp8_fp8, amdgcn_wmma_f32_16x16x64_fp8_bf8,
2830 amdgcn_wmma_f32_16x16x64_bf8_fp8, amdgcn_wmma_f32_16x16x64_bf8_bf8,
2831 amdgcn_wmma_f16_16x16x64_fp8_fp8, amdgcn_wmma_f16_16x16x64_fp8_bf8,
2832 amdgcn_wmma_f16_16x16x64_bf8_fp8, amdgcn_wmma_f16_16x16x64_bf8_bf8,
2833 amdgcn_wmma_f16_16x16x128_fp8_fp8, amdgcn_wmma_f16_16x16x128_fp8_bf8,
2834 amdgcn_wmma_f16_16x16x128_bf8_fp8, amdgcn_wmma_f16_16x16x128_bf8_bf8,
2835 amdgcn_wmma_f32_16x16x128_fp8_fp8, amdgcn_wmma_f32_16x16x128_fp8_bf8,
2836 amdgcn_wmma_f32_16x16x128_bf8_fp8, amdgcn_wmma_f32_16x16x128_bf8_bf8,
2837 amdgcn_wmma_i32_16x16x64_iu8, amdgcn_wmma_f32_16x16x128_f8f6f4,
2838 amdgcn_wmma_scale_f32_16x16x128_f8f6f4,
2839 amdgcn_wmma_scale16_f32_16x16x128_f8f6f4, amdgcn_wmma_f32_32x16x128_f4,
2840 amdgcn_wmma_scale_f32_32x16x128_f4, amdgcn_wmma_scale16_f32_32x16x128_f4,
2841 // WMMA GFX1251
2842 amdgcn_wmma_f64_16x16x4_f64,
2843 // SWMMAC GFX12
2844 amdgcn_swmmac_f32_16x16x32_f16, amdgcn_swmmac_f32_16x16x32_bf16,
2845 amdgcn_swmmac_f16_16x16x32_f16, amdgcn_swmmac_bf16_16x16x32_bf16,
2846 amdgcn_swmmac_i32_16x16x32_iu8, amdgcn_swmmac_i32_16x16x32_iu4,
2847 amdgcn_swmmac_i32_16x16x64_iu4, amdgcn_swmmac_f32_16x16x32_fp8_fp8,
2848 amdgcn_swmmac_f32_16x16x32_fp8_bf8, amdgcn_swmmac_f32_16x16x32_bf8_fp8,
2849 amdgcn_swmmac_f32_16x16x32_bf8_bf8,
2850 // SWMMAC GFX1250
2851 amdgcn_swmmac_f32_16x16x64_f16, amdgcn_swmmac_f32_16x16x64_bf16,
2852 amdgcn_swmmac_f16_16x16x64_f16, amdgcn_swmmac_bf16_16x16x64_bf16,
2853 amdgcn_swmmac_bf16f32_16x16x64_bf16, amdgcn_swmmac_f32_16x16x128_fp8_fp8,
2854 amdgcn_swmmac_f32_16x16x128_fp8_bf8, amdgcn_swmmac_f32_16x16x128_bf8_fp8,
2855 amdgcn_swmmac_f32_16x16x128_bf8_bf8, amdgcn_swmmac_f16_16x16x128_fp8_fp8,
2856 amdgcn_swmmac_f16_16x16x128_fp8_bf8, amdgcn_swmmac_f16_16x16x128_bf8_fp8,
2857 amdgcn_swmmac_f16_16x16x128_bf8_bf8, amdgcn_swmmac_i32_16x16x128_iu8})
2858 .Any(Init: {.Predicate: {}, .OperandMapping: {{}, {}, ApplyAllVgpr}});
2859
2860} // end initialize rules
2861