1//===- SPIRVLegalizerInfo.cpp --- SPIR-V Legalization Rules ------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the targeting of the Machinelegalizer class for SPIR-V.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SPIRVLegalizerInfo.h"
14#include "SPIRV.h"
15#include "SPIRVGlobalRegistry.h"
16#include "SPIRVSubtarget.h"
17#include "SPIRVUtils.h"
18#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
19#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
20#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
21#include "llvm/CodeGen/MachineInstr.h"
22#include "llvm/CodeGen/MachineRegisterInfo.h"
23#include "llvm/CodeGen/TargetOpcodes.h"
24#include "llvm/IR/IntrinsicsSPIRV.h"
25#include "llvm/Support/Debug.h"
26#include "llvm/Support/MathExtras.h"
27
28using namespace llvm;
29using namespace llvm::LegalizeActions;
30using namespace llvm::LegalityPredicates;
31
32#define DEBUG_TYPE "spirv-legalizer"
33
34LegalityPredicate typeOfExtendedScalars(unsigned TypeIdx, bool IsExtendedInts) {
35 return [IsExtendedInts, TypeIdx](const LegalityQuery &Query) {
36 const LLT Ty = Query.Types[TypeIdx];
37 return IsExtendedInts && Ty.isValid() && Ty.isScalar();
38 };
39}
40
41LegalityPredicate typeOfLongVectors(unsigned TypeIdx, bool IsLongVecs) {
42 return [TypeIdx, IsLongVecs](const LegalityQuery &Query) {
43 const LLT Ty = Query.Types[TypeIdx];
44 return IsLongVecs && Ty.isValid() && Ty.isVector();
45 };
46}
47
48SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) {
49 using namespace TargetOpcode;
50
51 this->ST = &ST;
52 GR = ST.getSPIRVGlobalRegistry();
53
54 const LLT s1 = LLT::scalar(SizeInBits: 1);
55 const LLT s8 = LLT::scalar(SizeInBits: 8);
56 const LLT s16 = LLT::scalar(SizeInBits: 16);
57 const LLT s32 = LLT::scalar(SizeInBits: 32);
58 const LLT s64 = LLT::scalar(SizeInBits: 64);
59 const LLT s128 = LLT::scalar(SizeInBits: 128);
60
61 const LLT v16s64 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 64);
62 const LLT v16s32 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 32);
63 const LLT v16s16 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 16);
64 const LLT v16s8 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 8);
65 const LLT v16s1 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 1);
66
67 const LLT v8s64 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 64);
68 const LLT v8s32 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 32);
69 const LLT v8s16 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16);
70 const LLT v8s8 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 8);
71 const LLT v8s1 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 1);
72
73 const LLT v4s64 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 64);
74 const LLT v4s32 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32);
75 const LLT v4s16 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16);
76 const LLT v4s8 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 8);
77 const LLT v4s1 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 1);
78
79 const LLT v3s64 = LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 64);
80 const LLT v3s32 = LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 32);
81 const LLT v3s16 = LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 16);
82 const LLT v3s8 = LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 8);
83 const LLT v3s1 = LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 1);
84
85 const LLT v2s64 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64);
86 const LLT v2s32 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32);
87 const LLT v2s16 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16);
88 const LLT v2s8 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 8);
89 const LLT v2s1 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 1);
90
91 const unsigned PSize = ST.getPointerSize();
92 const LLT p0 = LLT::pointer(AddressSpace: 0, SizeInBits: PSize); // Function
93 const LLT p1 = LLT::pointer(AddressSpace: 1, SizeInBits: PSize); // CrossWorkgroup
94 const LLT p2 = LLT::pointer(AddressSpace: 2, SizeInBits: PSize); // UniformConstant
95 const LLT p3 = LLT::pointer(AddressSpace: 3, SizeInBits: PSize); // Workgroup
96 const LLT p4 = LLT::pointer(AddressSpace: 4, SizeInBits: PSize); // Generic
97 const LLT p5 =
98 LLT::pointer(AddressSpace: 5, SizeInBits: PSize); // Input, SPV_INTEL_usm_storage_classes (Device)
99 const LLT p6 = LLT::pointer(AddressSpace: 6, SizeInBits: PSize); // SPV_INTEL_usm_storage_classes (Host)
100 const LLT p7 = LLT::pointer(AddressSpace: 7, SizeInBits: PSize); // Input
101 const LLT p8 = LLT::pointer(AddressSpace: 8, SizeInBits: PSize); // Output
102 const LLT p9 =
103 LLT::pointer(AddressSpace: 9, SizeInBits: PSize); // CodeSectionINTEL, SPV_INTEL_function_pointers
104 const LLT p10 = LLT::pointer(AddressSpace: 10, SizeInBits: PSize); // Private
105 const LLT p11 = LLT::pointer(AddressSpace: 11, SizeInBits: PSize); // StorageBuffer
106 const LLT p12 = LLT::pointer(AddressSpace: 12, SizeInBits: PSize); // Uniform
107 const LLT p13 = LLT::pointer(AddressSpace: 13, SizeInBits: PSize); // PushConstant
108
109 // TODO: remove copy-pasting here by using concatenation in some way.
110 auto allPtrsScalarsAndVectors = {
111 p0, p1, p2, p3, p4, p5, p6, p7, p8,
112 p9, p10, p11, p12, p13, s1, s8, s16, s32,
113 s64, s128, v2s1, v2s8, v2s16, v2s32, v2s64, v3s1, v3s8,
114 v3s16, v3s32, v3s64, v4s1, v4s8, v4s16, v4s32, v4s64, v8s1,
115 v8s8, v8s16, v8s32, v8s64, v16s1, v16s8, v16s16, v16s32, v16s64};
116
117 auto allVectors = {v2s1, v2s8, v2s16, v2s32, v2s64, v3s1, v3s8,
118 v3s16, v3s32, v3s64, v4s1, v4s8, v4s16, v4s32,
119 v4s64, v8s1, v8s8, v8s16, v8s32, v8s64, v16s1,
120 v16s8, v16s16, v16s32, v16s64};
121
122 auto allShaderVectors = {v2s1, v2s8, v2s16, v2s32, v2s64,
123 v3s1, v3s8, v3s16, v3s32, v3s64,
124 v4s1, v4s8, v4s16, v4s32, v4s64};
125
126 auto allScalars = {s1, s8, s16, s32, s64};
127
128 auto allScalarsAndVectors = {
129 s1, s8, s16, s32, s64, s128, v2s1, v2s8,
130 v2s16, v2s32, v2s64, v3s1, v3s8, v3s16, v3s32, v3s64,
131 v4s1, v4s8, v4s16, v4s32, v4s64, v8s1, v8s8, v8s16,
132 v8s32, v8s64, v16s1, v16s8, v16s16, v16s32, v16s64};
133
134 auto allShaderScalarsAndVectors = {
135 s1, s8, s16, s32, s64, s128, v2s1, v2s8, v2s16, v2s32, v2s64,
136 v3s1, v3s8, v3s16, v3s32, v3s64, v4s1, v4s8, v4s16, v4s32, v4s64};
137
138 auto &allowedScalarsAndVectors =
139 ST.isShader() ? allShaderScalarsAndVectors : allScalarsAndVectors;
140
141 auto allIntScalarsAndVectors = {
142 s8, s16, s32, s64, s128, v2s8, v2s16, v2s32, v2s64,
143 v3s8, v3s16, v3s32, v3s64, v4s8, v4s16, v4s32, v4s64, v8s8,
144 v8s16, v8s32, v8s64, v16s8, v16s16, v16s32, v16s64};
145
146 auto allBoolScalarsAndVectors = {s1, v2s1, v3s1, v4s1, v8s1, v16s1};
147 auto allBoolVectors = {v2s1, v3s1, v4s1, v8s1, v16s1};
148
149 auto allIntScalars = {s8, s16, s32, s64, s128};
150
151 auto allShaderIntVectors = {v2s8, v2s16, v2s32, v2s64, v3s8, v3s16,
152 v3s32, v3s64, v4s8, v4s16, v4s32, v4s64};
153
154 auto allIntVectors = {v2s8, v2s16, v2s32, v2s64, v3s8, v3s16, v3s32,
155 v3s64, v4s8, v4s16, v4s32, v4s64, v8s8, v8s16,
156 v8s32, v8s64, v16s8, v16s16, v16s32, v16s64};
157
158 auto &allowedIntVectorTypes =
159 ST.isShader() ? allShaderIntVectors : allIntVectors;
160
161 auto allFloatScalarsAndF16Vector2AndVector4s = {s16, s32, s64, v2s16, v4s16};
162
163 auto allFloatScalars = {s16, s32, s64};
164
165 auto allFloatScalarsAndVectors = {
166 s16, s32, s64, v2s16, v2s32, v2s64, v3s16, v3s32, v3s64,
167 v4s16, v4s32, v4s64, v8s16, v8s32, v8s64, v16s16, v16s32, v16s64};
168
169 auto allShaderFloatVectors = {v2s16, v2s32, v2s64, v3s16, v3s32,
170 v3s64, v4s16, v4s32, v4s64};
171
172 auto allFloatVectors = {v2s16, v2s32, v2s64, v3s16, v3s32,
173 v3s64, v4s16, v4s32, v4s64, v8s16,
174 v8s32, v8s64, v16s16, v16s32, v16s64};
175
176 auto &allowedFloatVectorTypes =
177 ST.isShader() ? allShaderFloatVectors : allFloatVectors;
178
179 auto allFloatAndIntScalarsAndPtrs = {s8, s16, s32, s64, p0, p1,
180 p2, p3, p4, p5, p6, p7,
181 p8, p9, p10, p11, p12, p13};
182
183 auto allPtrs = {p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13};
184
185 auto &allowedVectorTypes = ST.isShader() ? allShaderVectors : allVectors;
186
187 bool HasArbitraryPrecisionInts = ST.canUseExtension(
188 E: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers);
189 bool IsExtendedInts =
190 HasArbitraryPrecisionInts ||
191 ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_bit_instructions) ||
192 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_int4);
193 bool IsLongVecs = ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_long_vector);
194 auto ExtendedIntScalarsAndVectors =
195 [IsExtendedInts](const LegalityQuery &Query) {
196 const LLT Ty = Query.Types[0];
197 return IsExtendedInts && Ty.isValid() &&
198 !Ty.isPointerOrPointerVector() && Ty.getScalarSizeInBits() > 1;
199 };
200 auto ExtendedScalarsAndVectorsProduct = [IsExtendedInts](
201 const LegalityQuery &Query) {
202 const LLT Ty1 = Query.Types[0], Ty2 = Query.Types[1];
203 return IsExtendedInts && Ty1.isValid() && Ty2.isValid() &&
204 !Ty1.isPointerOrPointerVector() && !Ty2.isPointerOrPointerVector();
205 };
206 auto ExtendedPtrsScalarsAndVectors =
207 [IsExtendedInts](const LegalityQuery &Query) {
208 const LLT Ty = Query.Types[0];
209 return IsExtendedInts && Ty.isValid();
210 };
211
212 // The universal validation rules in the SPIR-V specification state that
213 // vector sizes are typically limited to 2, 3, or 4. However, larger vector
214 // sizes (8 and 16) are enabled when the Kernel capability is present. For
215 // shader execution models, vector sizes are strictly limited to 4. In
216 // non-shader contexts, vector sizes of 8 and 16 are also permitted, but
217 // arbitrary sizes (e.g., 6 or 11) are not.
218 uint32_t MaxVectorSize = ST.isShader() ? 4 : 16;
219 LLVM_DEBUG(dbgs() << "MaxVectorSize: " << MaxVectorSize << "\n");
220
221 for (auto Opc : getTypeFoldingSupportedOpcodes()) {
222 switch (Opc) {
223 case G_EXTRACT_VECTOR_ELT:
224 case G_UREM:
225 case G_SREM:
226 case G_UDIV:
227 case G_SDIV:
228 case G_FREM:
229 case G_SELECT:
230 break;
231 default:
232 getActionDefinitionsBuilder(Opcode: Opc)
233 .customFor(Types: allScalars)
234 .customFor(Types: allowedVectorTypes)
235 .customIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
236 .moreElementsToNextPow2(TypeIdx: 0)
237 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
238 Mutation: LegalizeMutations::changeElementCountTo(
239 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
240 .custom();
241 break;
242 }
243 }
244
245 getActionDefinitionsBuilder(Opcodes: {G_UREM, G_SREM, G_SDIV, G_UDIV, G_FREM})
246 .customFor(Types: allScalars)
247 .customFor(Types: allowedVectorTypes)
248 .customIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
249 .scalarizeIf(Predicate: numElementsNotPow2(TypeIdx: 0), TypeIdx: 0)
250 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
251 Mutation: LegalizeMutations::changeElementCountTo(
252 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
253 .custom();
254
255 getActionDefinitionsBuilder(Opcode: G_SELECT)
256 .customFor(Types: allScalars)
257 .customFor(Types: allowedVectorTypes)
258 .customIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
259 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
260 Mutation: LegalizeMutations::changeElementCountTo(
261 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
262 .custom();
263
264 getActionDefinitionsBuilder(Opcodes: {G_FMA, G_STRICT_FMA})
265 .legalFor(Types: allScalars)
266 .legalFor(Types: allowedVectorTypes)
267 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
268 .moreElementsToNextPow2(TypeIdx: 0)
269 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
270 Mutation: LegalizeMutations::changeElementCountTo(
271 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
272 .alwaysLegal();
273
274 getActionDefinitionsBuilder(Opcode: G_INTRINSIC_W_SIDE_EFFECTS).custom();
275
276 getActionDefinitionsBuilder(Opcode: G_SHUFFLE_VECTOR)
277 .legalForCartesianProduct(Types0: allowedVectorTypes, Types1: allowedVectorTypes)
278 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
279 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 1, IsLongVecs))
280 .moreElementsToNextPow2(TypeIdx: 0)
281 .lowerIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize))
282 .moreElementsToNextPow2(TypeIdx: 1)
283 .lowerIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 1, Size: MaxVectorSize));
284
285 getActionDefinitionsBuilder(Opcode: G_EXTRACT_VECTOR_ELT)
286 .customIf(Predicate: typeOfLongVectors(TypeIdx: 1, IsLongVecs))
287 .moreElementsToNextPow2(TypeIdx: 1)
288 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 1, Size: MaxVectorSize),
289 Mutation: LegalizeMutations::changeElementCountTo(
290 TypeIdx: 1, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
291 .custom();
292
293 getActionDefinitionsBuilder(Opcode: G_INSERT_VECTOR_ELT)
294 .customIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
295 .moreElementsToNextPow2(TypeIdx: 0)
296 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
297 Mutation: LegalizeMutations::changeElementCountTo(
298 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
299 .custom();
300
301 // Illegal G_UNMERGE_VALUES instructions should be handled
302 // during the combine phase.
303 getActionDefinitionsBuilder(Opcode: G_BUILD_VECTOR)
304 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
305 .legalIf(Predicate: vectorElementCountIsLessThanOrEqualTo(TypeIdx: 0, Size: MaxVectorSize))
306 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
307 Mutation: LegalizeMutations::changeElementCountTo(
308 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)));
309
310 // When entering the legalizer, there should be no G_BITCAST instructions.
311 // They should all be calls to the `spv_bitcast` intrinsic. The call to
312 // the intrinsic will be converted to a G_BITCAST during legalization if
313 // the vectors are not legal. After using the rules to legalize a G_BITCAST,
314 // we turn it back into a call to the intrinsic with a custom rule to avoid
315 // potential machine verifier failures.
316 getActionDefinitionsBuilder(Opcode: G_BITCAST)
317 .customIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
318 .moreElementsToNextPow2(TypeIdx: 0)
319 .moreElementsToNextPow2(TypeIdx: 1)
320 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
321 Mutation: LegalizeMutations::changeElementCountTo(
322 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
323 .lowerIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 1, Size: MaxVectorSize))
324 .custom();
325
326 // If the result is still illegal, the combiner should be able to remove it.
327 getActionDefinitionsBuilder(Opcode: G_CONCAT_VECTORS)
328 .legalForCartesianProduct(Types0: allowedVectorTypes, Types1: allowedVectorTypes)
329 .legalIf(Predicate: LegalityPredicates::any(P0: typeOfLongVectors(TypeIdx: 0, IsLongVecs),
330 P1: typeOfLongVectors(TypeIdx: 1, IsLongVecs)));
331
332 getActionDefinitionsBuilder(Opcode: G_SPLAT_VECTOR)
333 .legalFor(Types: allowedVectorTypes)
334 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
335 .moreElementsToNextPow2(TypeIdx: 0)
336 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
337 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx: 0, FromTypeIdx: MaxVectorSize))
338 .alwaysLegal();
339
340 // Vector Reduction Operations
341 getActionDefinitionsBuilder(
342 Opcodes: {G_VECREDUCE_SMIN, G_VECREDUCE_SMAX, G_VECREDUCE_UMIN, G_VECREDUCE_UMAX,
343 G_VECREDUCE_ADD, G_VECREDUCE_MUL, G_VECREDUCE_FMUL, G_VECREDUCE_FMIN,
344 G_VECREDUCE_FMAX, G_VECREDUCE_FMINIMUM, G_VECREDUCE_FMAXIMUM,
345 G_VECREDUCE_OR, G_VECREDUCE_AND, G_VECREDUCE_XOR})
346 .legalFor(Types: allowedVectorTypes)
347 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
348 .scalarize(TypeIdx: 1)
349 .lower();
350
351 getActionDefinitionsBuilder(Opcodes: {G_VECREDUCE_SEQ_FADD, G_VECREDUCE_SEQ_FMUL})
352 .scalarize(TypeIdx: 2)
353 .lower();
354
355 // Illegal G_UNMERGE_VALUES instructions should be handled
356 // during the combine phase.
357 getActionDefinitionsBuilder(Opcode: G_UNMERGE_VALUES)
358 .legalIf(Predicate: LegalityPredicates::any(P0: typeOfLongVectors(TypeIdx: 0, IsLongVecs),
359 P1: typeOfLongVectors(TypeIdx: 1, IsLongVecs)))
360 .legalIf(Predicate: vectorElementCountIsLessThanOrEqualTo(TypeIdx: 1, Size: MaxVectorSize));
361
362 getActionDefinitionsBuilder(Opcodes: {G_MEMCPY, G_MEMCPY_INLINE, G_MEMMOVE})
363 .unsupportedIf(Predicate: LegalityPredicates::any(P0: typeIs(TypeIdx: 0, TypesInit: p9), P1: typeIs(TypeIdx: 1, TypesInit: p9)))
364 .legalIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allPtrs), P1: typeInSet(TypeIdx: 1, TypesInit: allPtrs)));
365
366 getActionDefinitionsBuilder(Opcodes: {G_MEMSET, G_MEMSET_INLINE})
367 .unsupportedIf(Predicate: typeIs(TypeIdx: 0, TypesInit: p9))
368 .legalIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allPtrs), P1: typeInSet(TypeIdx: 1, TypesInit: allIntScalars)));
369
370 getActionDefinitionsBuilder(Opcode: G_ADDRSPACE_CAST)
371 .legalForCartesianProduct(Types0: allPtrs, Types1: allPtrs);
372
373 // Should we be legalizing bad scalar sizes like s5 here instead
374 // of handling them in the instruction selector?
375 getActionDefinitionsBuilder(Opcodes: {G_LOAD, G_STORE})
376 .unsupportedIf(Predicate: typeIs(TypeIdx: 1, TypesInit: p9))
377 .legalForCartesianProduct(Types0: allowedVectorTypes, Types1: allPtrs)
378 .legalForCartesianProduct(Types0: allPtrs, Types1: allPtrs)
379 .legalIf(Predicate: isScalar(TypeIdx: 0))
380 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
381 .custom();
382
383 getActionDefinitionsBuilder(Opcodes: {G_SMIN, G_SMAX, G_UMIN, G_UMAX, G_ABS,
384 G_BITREVERSE, G_SADDSAT, G_UADDSAT, G_SSUBSAT,
385 G_USUBSAT, G_SCMP, G_UCMP})
386 .legalFor(Types: allIntScalars)
387 .legalFor(Types: allowedIntVectorTypes)
388 .legalIf(Predicate: ExtendedIntScalarsAndVectors)
389 // LLVM i1 maps to OpTypeBool, not OpTypeInt.
390 .scalarizeIf(Predicate: typeInSet(TypeIdx: 0, TypesInit: allBoolVectors), TypeIdx: 0)
391 .minScalar(TypeIdx: 0, Ty: s32)
392 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
393 Mutation: LegalizeMutations::changeElementCountTo(
394 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
395 .moreElementsToNextPow2(TypeIdx: 0);
396
397 getActionDefinitionsBuilder(Opcodes: {G_SSHLSAT, G_USHLSAT}).lower();
398
399 getActionDefinitionsBuilder(Opcodes: {G_FLDEXP, G_STRICT_FLDEXP})
400 .legalForCartesianProduct(Types0: allFloatScalarsAndVectors, Types1: allIntScalars);
401
402 getActionDefinitionsBuilder(Opcodes: {G_FPTOSI, G_FPTOUI})
403 .legalForCartesianProduct(Types0: allIntScalarsAndVectors,
404 Types1: allFloatScalarsAndVectors);
405
406 getActionDefinitionsBuilder(Opcodes: {G_FPTOSI_SAT, G_FPTOUI_SAT})
407 .legalForCartesianProduct(Types0: allIntScalarsAndVectors,
408 Types1: allFloatScalarsAndVectors);
409
410 getActionDefinitionsBuilder(Opcodes: {G_SITOFP, G_UITOFP})
411 .legalForCartesianProduct(Types0: allFloatScalarsAndVectors,
412 Types1: allScalarsAndVectors);
413
414 getActionDefinitionsBuilder(Opcode: G_CTPOP)
415 .legalForCartesianProduct(Types: allIntScalarsAndVectors)
416 .legalIf(Predicate: ExtendedScalarsAndVectorsProduct)
417 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs));
418
419 getActionDefinitionsBuilder(Opcodes: {G_TRUNC, G_ZEXT, G_SEXT, G_ANYEXT})
420 .legalForCartesianProduct(Types: allowedScalarsAndVectors)
421 .legalIf(Predicate: ExtendedScalarsAndVectorsProduct)
422 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
423 .moreElementsToNextPow2(TypeIdx: 0)
424 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
425 Mutation: LegalizeMutations::changeElementCountTo(
426 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)));
427
428 getActionDefinitionsBuilder(Opcode: G_SEXT_INREG)
429 .lowerIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
430 .moreElementsToNextPow2(TypeIdx: 0)
431 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
432 Mutation: LegalizeMutations::changeElementCountTo(
433 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
434 .lower();
435
436 getActionDefinitionsBuilder(Opcode: G_PHI)
437 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
438 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
439 Mutation: LegalizeMutations::changeElementCountTo(
440 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
441 .legalFor(Types: allPtrsScalarsAndVectors)
442 .legalIf(Predicate: ExtendedPtrsScalarsAndVectors)
443 .moreElementsToNextPow2(TypeIdx: 0);
444
445 getActionDefinitionsBuilder(Opcode: G_BITCAST).legalIf(
446 Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allPtrsScalarsAndVectors),
447 P1: typeInSet(TypeIdx: 1, TypesInit: allPtrsScalarsAndVectors)));
448
449 getActionDefinitionsBuilder(Opcodes: {G_IMPLICIT_DEF, G_FREEZE})
450 .legalFor(Types: {s1, s128})
451 .legalFor(Types: allFloatAndIntScalarsAndPtrs)
452 .legalFor(Types: allowedVectorTypes)
453 .legalIf(Predicate: [](const LegalityQuery &Query) {
454 return Query.Types[0].isPointerVector();
455 })
456 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
457 .moreElementsToNextPow2(TypeIdx: 0)
458 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
459 Mutation: LegalizeMutations::changeElementCountTo(
460 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)));
461
462 getActionDefinitionsBuilder(Opcodes: {G_STACKSAVE, G_STACKRESTORE}).alwaysLegal();
463
464 getActionDefinitionsBuilder(Opcode: G_INTTOPTR)
465 .legalForCartesianProduct(Types0: allPtrs, Types1: allIntScalars)
466 .legalIf(
467 Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allPtrs), P1: typeOfExtendedScalars(TypeIdx: 1, IsExtendedInts)))
468 .legalIf(Predicate: [](const LegalityQuery &Query) {
469 const LLT DstTy = Query.Types[0];
470 const LLT SrcTy = Query.Types[1];
471 return DstTy.isPointerVector() && SrcTy.isVector() &&
472 !SrcTy.isPointer() &&
473 DstTy.getNumElements() == SrcTy.getNumElements();
474 });
475 getActionDefinitionsBuilder(Opcode: G_PTRTOINT)
476 .legalForCartesianProduct(Types0: allIntScalars, Types1: allPtrs)
477 .legalIf(
478 Predicate: all(P0: typeOfExtendedScalars(TypeIdx: 0, IsExtendedInts), P1: typeInSet(TypeIdx: 1, TypesInit: allPtrs)))
479 .legalIf(Predicate: [](const LegalityQuery &Query) {
480 const LLT DstTy = Query.Types[0];
481 const LLT SrcTy = Query.Types[1];
482 return SrcTy.isPointerVector() && DstTy.isVector() &&
483 !DstTy.isPointer() &&
484 DstTy.getNumElements() == SrcTy.getNumElements();
485 });
486 getActionDefinitionsBuilder(Opcode: G_PTR_ADD)
487 .legalForCartesianProduct(Types0: allPtrs, Types1: allIntScalars)
488 .legalIf(
489 Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allPtrs), P1: typeOfExtendedScalars(TypeIdx: 1, IsExtendedInts)));
490
491 getActionDefinitionsBuilder(Opcode: G_PTRMASK)
492 .legalForCartesianProduct(Types0: allPtrs, Types1: allIntScalars)
493 .legalIf(
494 Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allPtrs), P1: typeOfExtendedScalars(TypeIdx: 1, IsExtendedInts)))
495 .legalIf(Predicate: [](const LegalityQuery &Query) {
496 const LLT PtrTy = Query.Types[0];
497 const LLT MaskTy = Query.Types[1];
498 return PtrTy.isPointerVector() && MaskTy.isVector() &&
499 !MaskTy.isPointer() &&
500 PtrTy.getNumElements() == MaskTy.getNumElements();
501 });
502
503 // ST.canDirectlyComparePointers() for pointer args is supported in
504 // legalizeCustom().
505 getActionDefinitionsBuilder(Opcode: G_ICMP)
506 .unsupportedIf(Predicate: LegalityPredicates::any(
507 P0: all(P0: typeIs(TypeIdx: 0, TypesInit: p9), P1: typeInSet(TypeIdx: 1, TypesInit: allPtrs), args: typeIsNot(TypeIdx: 1, Type: p9)),
508 P1: all(P0: typeInSet(TypeIdx: 0, TypesInit: allPtrs), P1: typeIsNot(TypeIdx: 0, Type: p9), args: typeIs(TypeIdx: 1, TypesInit: p9))))
509 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 1, Size: MaxVectorSize),
510 Mutation: LegalizeMutations::changeElementCountTo(
511 TypeIdx: 1, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
512 .legalIf(Predicate: [IsExtendedInts](const LegalityQuery &Query) {
513 const LLT Ty = Query.Types[1];
514 return IsExtendedInts && Ty.isValid() && !Ty.isPointerOrPointerVector();
515 })
516 .customIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allBoolScalarsAndVectors),
517 P1: typeInSet(TypeIdx: 1, TypesInit: allPtrsScalarsAndVectors)));
518
519 getActionDefinitionsBuilder(Opcode: G_FCMP)
520 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 1, Size: MaxVectorSize),
521 Mutation: LegalizeMutations::changeElementCountTo(
522 TypeIdx: 1, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
523 .legalIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allBoolScalarsAndVectors),
524 P1: typeInSet(TypeIdx: 1, TypesInit: allFloatScalarsAndVectors)))
525 .moreElementsToNextPow2(TypeIdx: 1);
526
527 getActionDefinitionsBuilder(Opcodes: {G_ATOMICRMW_OR, G_ATOMICRMW_ADD, G_ATOMICRMW_AND,
528 G_ATOMICRMW_MAX, G_ATOMICRMW_MIN,
529 G_ATOMICRMW_SUB, G_ATOMICRMW_XOR,
530 G_ATOMICRMW_UMAX, G_ATOMICRMW_UMIN})
531 .legalForCartesianProduct(Types0: allIntScalars, Types1: allPtrs);
532
533 getActionDefinitionsBuilder(
534 Opcodes: {G_ATOMICRMW_FADD, G_ATOMICRMW_FSUB, G_ATOMICRMW_FMIN, G_ATOMICRMW_FMAX})
535 .legalForCartesianProduct(Types0: allFloatScalarsAndF16Vector2AndVector4s,
536 Types1: allPtrs);
537
538 getActionDefinitionsBuilder(Opcode: G_ATOMICRMW_XCHG)
539 .legalForCartesianProduct(Types0: allFloatAndIntScalarsAndPtrs, Types1: allPtrs);
540
541 getActionDefinitionsBuilder(Opcode: G_ATOMIC_CMPXCHG_WITH_SUCCESS).lower();
542 // TODO: add proper legalization rules.
543 getActionDefinitionsBuilder(Opcode: G_ATOMIC_CMPXCHG).alwaysLegal();
544 getActionDefinitionsBuilder(Opcode: G_PREFETCH).alwaysLegal();
545
546 getActionDefinitionsBuilder(Opcodes: {G_UADDO, G_USUBO, G_UMULO, G_SMULO})
547 .alwaysLegal();
548
549 getActionDefinitionsBuilder(Opcodes: {G_SADDO, G_SSUBO}).lower();
550
551 // Lowering widens s64 to s128, which needs
552 // SPV_ALTERA_arbitrary_precision_integers. Mark s64 unsupported otherwise.
553 auto &MulFix = getActionDefinitionsBuilder(Opcodes: {G_SMULFIX, G_UMULFIX});
554 if (!HasArbitraryPrecisionInts)
555 MulFix.unsupportedFor(Types: {s64});
556 MulFix.lower();
557
558 getActionDefinitionsBuilder(Opcodes: {G_LROUND, G_LLROUND})
559 .legalForCartesianProduct(Types0: allIntScalarsAndVectors,
560 Types1: allFloatScalarsAndVectors);
561
562 // FP conversions.
563 getActionDefinitionsBuilder(Opcodes: {G_FPTRUNC, G_FPEXT})
564 .legalForCartesianProduct(Types: allFloatScalarsAndVectors);
565
566 // Pointer-handling.
567 getActionDefinitionsBuilder(Opcode: G_FRAME_INDEX).legalFor(Types: {p0});
568
569 getActionDefinitionsBuilder(Opcode: G_GLOBAL_VALUE).legalFor(Types: allPtrs);
570
571 // Control-flow. In some cases (e.g. constants) s1 may be promoted to s32.
572 getActionDefinitionsBuilder(Opcode: G_BR).alwaysLegal();
573 getActionDefinitionsBuilder(Opcode: G_BRCOND).legalFor(Types: {s1, s32});
574
575 getActionDefinitionsBuilder(Opcode: G_FFREXP).legalForCartesianProduct(
576 Types0: allFloatScalarsAndVectors, Types1: {s32, v2s32, v3s32, v4s32, v8s32, v16s32});
577
578 // TODO: Review the target OpenCL and GLSL Extended Instruction Set specs to
579 // tighten these requirements. Many of these math functions are only legal on
580 // specific bitwidths, so they are not selectable for
581 // allFloatScalarsAndVectors.
582 // clang-format off
583 getActionDefinitionsBuilder(Opcodes: {G_STRICT_FSQRT,
584 G_FPOW,
585 G_FEXP,
586 G_FMODF,
587 G_FSINCOS,
588 G_FEXP2,
589 G_FEXP10,
590 G_FLOG,
591 G_FLOG2,
592 G_FLOG10,
593 G_FABS,
594 G_FMINNUM,
595 G_FMAXNUM,
596 G_FCEIL,
597 G_FCOS,
598 G_FSIN,
599 G_FTAN,
600 G_FACOS,
601 G_FASIN,
602 G_FATAN,
603 G_FATAN2,
604 G_FCOSH,
605 G_FSINH,
606 G_FTANH,
607 G_FSQRT,
608 G_FFLOOR,
609 G_FRINT,
610 G_FNEARBYINT,
611 G_INTRINSIC_ROUND,
612 G_INTRINSIC_TRUNC,
613 G_INTRINSIC_ROUNDEVEN})
614 .legalFor(Types: allFloatScalars)
615 .legalFor(Types: allowedFloatVectorTypes)
616 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
617 Mutation: LegalizeMutations::changeElementCountTo(
618 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
619 .moreElementsToNextPow2(TypeIdx: 0);
620 // clang-format on
621
622 getActionDefinitionsBuilder(Opcode: G_FCOPYSIGN)
623 .legalForCartesianProduct(Types0: allFloatScalarsAndVectors,
624 Types1: allFloatScalarsAndVectors)
625 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
626 Mutation: LegalizeMutations::changeElementCountTo(
627 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
628 .moreElementsToNextPow2(TypeIdx: 0);
629
630 getActionDefinitionsBuilder(Opcode: G_FPOWI).legalForCartesianProduct(
631 Types0: allFloatScalarsAndVectors, Types1: allIntScalarsAndVectors);
632
633 if (ST.canUseExtInstSet(E: SPIRV::InstructionSet::OpenCL_std)) {
634 getActionDefinitionsBuilder(
635 Opcodes: {G_CTTZ, G_CTTZ_ZERO_POISON, G_CTLZ, G_CTLZ_ZERO_POISON})
636 .legalForCartesianProduct(Types0: allIntScalarsAndVectors,
637 Types1: allIntScalarsAndVectors);
638
639 // Struct return types become a single scalar, so cannot easily legalize.
640 getActionDefinitionsBuilder(Opcodes: {G_SMULH, G_UMULH}).alwaysLegal();
641 }
642
643 getActionDefinitionsBuilder(Opcode: G_IS_FPCLASS).custom();
644
645 getActionDefinitionsBuilder(Opcodes: {G_INTRINSIC, G_INTRINSIC_CONVERGENT,
646 G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS})
647 .alwaysLegal();
648 getActionDefinitionsBuilder(Opcode: G_FENCE).alwaysLegal();
649 getActionDefinitionsBuilder(Opcodes: {G_TRAP, G_DEBUGTRAP, G_UBSANTRAP}).alwaysLegal();
650
651 verify(MII: *ST.getInstrInfo());
652}
653
654static bool legalizeExtractVectorElt(LegalizerHelper &Helper,
655 MachineInstr &MI) {
656 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
657 Register DstReg = MI.getOperand(i: 0).getReg();
658 Register SrcReg = MI.getOperand(i: 1).getReg();
659 Register IdxReg = MI.getOperand(i: 2).getReg();
660
661 MIRBuilder
662 .buildIntrinsic(ID: Intrinsic::spv_extractelt, Res: ArrayRef<Register>{DstReg})
663 .addUse(RegNo: SrcReg)
664 .addUse(RegNo: IdxReg);
665 MI.eraseFromParent();
666 return true;
667}
668
669static bool legalizeInsertVectorElt(LegalizerHelper &Helper, MachineInstr &MI) {
670 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
671 Register DstReg = MI.getOperand(i: 0).getReg();
672 Register SrcReg = MI.getOperand(i: 1).getReg();
673 Register ValReg = MI.getOperand(i: 2).getReg();
674 Register IdxReg = MI.getOperand(i: 3).getReg();
675
676 MIRBuilder
677 .buildIntrinsic(ID: Intrinsic::spv_insertelt, Res: ArrayRef<Register>{DstReg})
678 .addUse(RegNo: SrcReg)
679 .addUse(RegNo: ValReg)
680 .addUse(RegNo: IdxReg);
681 MI.eraseFromParent();
682 return true;
683}
684
685static Register convertPtrToInt(Register Reg, LLT ConvTy, SPIRVTypeInst SpvType,
686 LegalizerHelper &Helper,
687 MachineRegisterInfo &MRI,
688 SPIRVGlobalRegistry *GR) {
689 Register ConvReg = MRI.createGenericVirtualRegister(Ty: ConvTy);
690 MRI.setRegClass(Reg: ConvReg, RC: GR->getRegClass(SpvType));
691 GR->assignSPIRVTypeToVReg(Type: SpvType, VReg: ConvReg, MF: Helper.MIRBuilder.getMF());
692 Helper.MIRBuilder.buildInstr(Opcode: TargetOpcode::G_PTRTOINT)
693 .addDef(RegNo: ConvReg)
694 .addUse(RegNo: Reg);
695 return ConvReg;
696}
697
698static bool needsVectorLegalization(const LLT &Ty, const SPIRVSubtarget &ST) {
699 if (!Ty.isVector() ||
700 ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_long_vector))
701 return false;
702 unsigned NumElements = Ty.getNumElements();
703 unsigned MaxVectorSize = ST.isShader() ? 4 : 16;
704 return (NumElements > 4 && !isPowerOf2_32(Value: NumElements)) ||
705 NumElements > MaxVectorSize;
706}
707
708static bool legalizeLoad(LegalizerHelper &Helper, MachineInstr &MI,
709 SPIRVGlobalRegistry *GR) {
710 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
711 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
712 Register DstReg = MI.getOperand(i: 0).getReg();
713 Register PtrReg = MI.getOperand(i: 1).getReg();
714 LLT DstTy = MRI.getType(Reg: DstReg);
715
716 if (!DstTy.isVector())
717 return true;
718
719 const SPIRVSubtarget &ST = MI.getMF()->getSubtarget<SPIRVSubtarget>();
720 if (!needsVectorLegalization(Ty: DstTy, ST))
721 return true;
722
723 SmallVector<Register, 8> SplitRegs;
724 LLT EltTy = DstTy.getElementType();
725 unsigned NumElts = DstTy.getNumElements();
726
727 LLT PtrTy = MRI.getType(Reg: PtrReg);
728 auto Zero = MIRBuilder.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: 0);
729
730 for (unsigned i = 0; i < NumElts; ++i) {
731 auto Idx = MIRBuilder.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: i);
732 Register EltPtr = MRI.createGenericVirtualRegister(Ty: PtrTy);
733
734 MIRBuilder.buildIntrinsic(ID: Intrinsic::spv_gep, Res: ArrayRef<Register>{EltPtr})
735 .addImm(Val: 1) // InBounds
736 .addUse(RegNo: PtrReg)
737 .addUse(RegNo: Zero.getReg(Idx: 0))
738 .addUse(RegNo: Idx.getReg(Idx: 0));
739
740 MachinePointerInfo EltPtrInfo;
741 Align EltAlign = Align(1);
742 if (!MI.memoperands_empty()) {
743 MachineMemOperand *MMO = *MI.memoperands_begin();
744 EltPtrInfo =
745 MMO->getPointerInfo().getWithOffset(O: i * EltTy.getSizeInBytes());
746 EltAlign = commonAlignment(A: MMO->getAlign(), Offset: i * EltTy.getSizeInBytes());
747 }
748
749 Register EltReg = MRI.createGenericVirtualRegister(Ty: EltTy);
750 MIRBuilder.buildLoad(Res: EltReg, Addr: EltPtr, PtrInfo: EltPtrInfo, Alignment: EltAlign);
751 SplitRegs.push_back(Elt: EltReg);
752 }
753
754 MIRBuilder.buildBuildVector(Res: DstReg, Ops: SplitRegs);
755 MI.eraseFromParent();
756 return true;
757}
758
759static bool legalizeStore(LegalizerHelper &Helper, MachineInstr &MI,
760 SPIRVGlobalRegistry *GR) {
761 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
762 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
763 Register ValReg = MI.getOperand(i: 0).getReg();
764 Register PtrReg = MI.getOperand(i: 1).getReg();
765 LLT ValTy = MRI.getType(Reg: ValReg);
766
767 assert(ValTy.isVector() && "Expected vector store");
768
769 SmallVector<Register, 8> SplitRegs;
770 LLT EltTy = ValTy.getElementType();
771 unsigned NumElts = ValTy.getNumElements();
772
773 for (unsigned i = 0; i < NumElts; ++i)
774 SplitRegs.push_back(Elt: MRI.createGenericVirtualRegister(Ty: EltTy));
775
776 MIRBuilder.buildUnmerge(Res: SplitRegs, Op: ValReg);
777
778 LLT PtrTy = MRI.getType(Reg: PtrReg);
779 auto Zero = MIRBuilder.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: 0);
780
781 for (unsigned i = 0; i < NumElts; ++i) {
782 auto Idx = MIRBuilder.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: i);
783 Register EltPtr = MRI.createGenericVirtualRegister(Ty: PtrTy);
784
785 MIRBuilder.buildIntrinsic(ID: Intrinsic::spv_gep, Res: ArrayRef<Register>{EltPtr})
786 .addImm(Val: 1) // InBounds
787 .addUse(RegNo: PtrReg)
788 .addUse(RegNo: Zero.getReg(Idx: 0))
789 .addUse(RegNo: Idx.getReg(Idx: 0));
790
791 MachinePointerInfo EltPtrInfo;
792 Align EltAlign = Align(1);
793 if (!MI.memoperands_empty()) {
794 MachineMemOperand *MMO = *MI.memoperands_begin();
795 EltPtrInfo =
796 MMO->getPointerInfo().getWithOffset(O: i * EltTy.getSizeInBytes());
797 EltAlign = commonAlignment(A: MMO->getAlign(), Offset: i * EltTy.getSizeInBytes());
798 }
799
800 MIRBuilder.buildStore(Val: SplitRegs[i], Addr: EltPtr, PtrInfo: EltPtrInfo, Alignment: EltAlign);
801 }
802
803 MI.eraseFromParent();
804 return true;
805}
806
807bool SPIRVLegalizerInfo::legalizeCustom(
808 LegalizerHelper &Helper, MachineInstr &MI,
809 LostDebugLocObserver &LocObserver) const {
810 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
811 switch (MI.getOpcode()) {
812 default:
813 // TODO: implement legalization for other opcodes.
814 return true;
815 case TargetOpcode::G_BITCAST:
816 return legalizeBitcast(Helper, MI);
817 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
818 return legalizeExtractVectorElt(Helper, MI);
819 case TargetOpcode::G_INSERT_VECTOR_ELT:
820 return legalizeInsertVectorElt(Helper, MI);
821 case TargetOpcode::G_INTRINSIC:
822 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
823 return legalizeIntrinsic(Helper, MI);
824 case TargetOpcode::G_IS_FPCLASS:
825 return legalizeIsFPClass(Helper, MI, LocObserver);
826 case TargetOpcode::G_ICMP: {
827 auto &Op0 = MI.getOperand(i: 2);
828 auto &Op1 = MI.getOperand(i: 3);
829 Register Reg0 = Op0.getReg();
830 Register Reg1 = Op1.getReg();
831 CmpInst::Predicate Cond =
832 static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
833 if ((!ST->canDirectlyComparePointers() ||
834 (Cond != CmpInst::ICMP_EQ && Cond != CmpInst::ICMP_NE)) &&
835 MRI.getType(Reg: Reg0).isPointer() && MRI.getType(Reg: Reg1).isPointer()) {
836 LLT ConvT = LLT::scalar(SizeInBits: ST->getPointerSize());
837 Type *LLVMTy = IntegerType::get(C&: MI.getMF()->getFunction().getContext(),
838 NumBits: ST->getPointerSize());
839 SPIRVTypeInst SpirvTy = GR->getOrCreateSPIRVType(
840 Type: LLVMTy, MIRBuilder&: Helper.MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
841 Op0.setReg(convertPtrToInt(Reg: Reg0, ConvTy: ConvT, SpvType: SpirvTy, Helper, MRI, GR));
842 Op1.setReg(convertPtrToInt(Reg: Reg1, ConvTy: ConvT, SpvType: SpirvTy, Helper, MRI, GR));
843 }
844 return true;
845 }
846 case TargetOpcode::G_LOAD:
847 return legalizeLoad(Helper, MI, GR);
848 case TargetOpcode::G_STORE:
849 return legalizeStore(Helper, MI, GR);
850 }
851}
852
853static MachineInstrBuilder
854createStackTemporaryForVector(LegalizerHelper &Helper, SPIRVGlobalRegistry *GR,
855 Register SrcReg, LLT SrcTy,
856 MachinePointerInfo &PtrInfo, Align &VecAlign) {
857 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
858 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
859
860 VecAlign = Helper.getStackTemporaryAlignment(Type: SrcTy);
861 auto StackTemp = Helper.createStackTemporary(
862 Bytes: TypeSize::getFixed(ExactSize: SrcTy.getSizeInBytes()), Alignment: VecAlign, PtrInfo);
863
864 // Set the type of StackTemp to a pointer to an array of the element type.
865 SPIRVTypeInst SpvSrcTy = GR->getSPIRVTypeForVReg(VReg: SrcReg);
866 SPIRVTypeInst EltSpvTy = GR->getScalarOrVectorComponentType(Type: SpvSrcTy);
867 const Type *LLVMEltTy = GR->getTypeForSPIRVType(Ty: EltSpvTy);
868 const Type *LLVMArrTy =
869 ArrayType::get(ElementType: const_cast<Type *>(LLVMEltTy), NumElements: SrcTy.getNumElements());
870 SPIRVTypeInst ArrSpvTy = GR->getOrCreateSPIRVType(
871 Type: LLVMArrTy, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
872 SPIRVTypeInst PtrToArrSpvTy = GR->getOrCreateSPIRVPointerType(
873 BaseType: ArrSpvTy, MIRBuilder, SC: SPIRV::StorageClass::Function);
874
875 Register StackReg = StackTemp.getReg(Idx: 0);
876 MRI.setRegClass(Reg: StackReg, RC: GR->getRegClass(SpvType: PtrToArrSpvTy));
877 GR->assignSPIRVTypeToVReg(Type: PtrToArrSpvTy, VReg: StackReg, MF: MIRBuilder.getMF());
878
879 return StackTemp;
880}
881
882static bool legalizeSpvBitcast(LegalizerHelper &Helper, MachineInstr &MI,
883 SPIRVGlobalRegistry *GR) {
884 LLVM_DEBUG(dbgs() << "Found a bitcast instruction\n");
885 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
886 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
887 const SPIRVSubtarget &ST = MI.getMF()->getSubtarget<SPIRVSubtarget>();
888
889 Register DstReg = MI.getOperand(i: 0).getReg();
890 Register SrcReg = MI.getOperand(i: 2).getReg();
891 LLT DstTy = MRI.getType(Reg: DstReg);
892 LLT SrcTy = MRI.getType(Reg: SrcReg);
893
894 // If an spv_bitcast needs to be legalized, we convert it to G_BITCAST to
895 // allow using the generic legalization rules.
896 if (needsVectorLegalization(Ty: DstTy, ST) ||
897 needsVectorLegalization(Ty: SrcTy, ST)) {
898 LLVM_DEBUG(dbgs() << "Replacing with a G_BITCAST\n");
899 MIRBuilder.buildBitcast(Dst: DstReg, Src: SrcReg);
900 MI.eraseFromParent();
901 }
902 return true;
903}
904
905static bool legalizeSpvInsertElt(LegalizerHelper &Helper, MachineInstr &MI,
906 SPIRVGlobalRegistry *GR) {
907 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
908 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
909 const SPIRVSubtarget &ST = MI.getMF()->getSubtarget<SPIRVSubtarget>();
910
911 Register DstReg = MI.getOperand(i: 0).getReg();
912 LLT DstTy = MRI.getType(Reg: DstReg);
913
914 if (needsVectorLegalization(Ty: DstTy, ST)) {
915 Register SrcReg = MI.getOperand(i: 2).getReg();
916 Register ValReg = MI.getOperand(i: 3).getReg();
917 LLT SrcTy = MRI.getType(Reg: SrcReg);
918 MachineOperand &IdxOperand = MI.getOperand(i: 4);
919
920 if (getImm(MO: IdxOperand, MRI: &MRI)) {
921 uint64_t IdxVal = foldImm(MO: IdxOperand, MRI: &MRI);
922 if (IdxVal < SrcTy.getNumElements()) {
923 SmallVector<Register, 8> Regs;
924 SPIRVTypeInst ElementType =
925 GR->getScalarOrVectorComponentType(Type: GR->getSPIRVTypeForVReg(VReg: DstReg));
926 LLT ElementLLTTy = GR->getRegType(SpvType: ElementType);
927 for (unsigned I = 0, E = SrcTy.getNumElements(); I < E; ++I) {
928 Register Reg = MRI.createGenericVirtualRegister(Ty: ElementLLTTy);
929 MRI.setRegClass(Reg, RC: GR->getRegClass(SpvType: ElementType));
930 GR->assignSPIRVTypeToVReg(Type: ElementType, VReg: Reg, MF: *MI.getMF());
931 Regs.push_back(Elt: Reg);
932 }
933 MIRBuilder.buildUnmerge(Res: Regs, Op: SrcReg);
934 Regs[IdxVal] = ValReg;
935 MIRBuilder.buildBuildVector(Res: DstReg, Ops: Regs);
936 MI.eraseFromParent();
937 return true;
938 }
939 }
940
941 LLT EltTy = SrcTy.getElementType();
942 Align VecAlign;
943 MachinePointerInfo PtrInfo;
944 auto StackTemp = createStackTemporaryForVector(Helper, GR, SrcReg, SrcTy,
945 PtrInfo, VecAlign);
946
947 MIRBuilder.buildStore(Val: SrcReg, Addr: StackTemp, PtrInfo, Alignment: VecAlign);
948
949 Register IdxReg = IdxOperand.getReg();
950 LLT PtrTy = MRI.getType(Reg: StackTemp.getReg(Idx: 0));
951 Register EltPtr = MRI.createGenericVirtualRegister(Ty: PtrTy);
952 auto Zero = MIRBuilder.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: 0);
953
954 MIRBuilder.buildIntrinsic(ID: Intrinsic::spv_gep, Res: ArrayRef<Register>{EltPtr})
955 .addImm(Val: 1) // InBounds
956 .addUse(RegNo: StackTemp.getReg(Idx: 0))
957 .addUse(RegNo: Zero.getReg(Idx: 0))
958 .addUse(RegNo: IdxReg);
959
960 MachinePointerInfo EltPtrInfo = MachinePointerInfo(PtrTy.getAddressSpace());
961 Align EltAlign = Helper.getStackTemporaryAlignment(Type: EltTy);
962 MIRBuilder.buildStore(Val: ValReg, Addr: EltPtr, PtrInfo: EltPtrInfo, Alignment: EltAlign);
963
964 MIRBuilder.buildLoad(Res: DstReg, Addr: StackTemp, PtrInfo, Alignment: VecAlign);
965 MI.eraseFromParent();
966 return true;
967 }
968 return true;
969}
970
971static bool legalizeSpvExtractElt(LegalizerHelper &Helper, MachineInstr &MI,
972 SPIRVGlobalRegistry *GR) {
973 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
974 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
975 const SPIRVSubtarget &ST = MI.getMF()->getSubtarget<SPIRVSubtarget>();
976
977 Register SrcReg = MI.getOperand(i: 2).getReg();
978 LLT SrcTy = MRI.getType(Reg: SrcReg);
979
980 if (needsVectorLegalization(Ty: SrcTy, ST)) {
981 Register DstReg = MI.getOperand(i: 0).getReg();
982 MachineOperand &IdxOperand = MI.getOperand(i: 3);
983
984 if (getImm(MO: IdxOperand, MRI: &MRI)) {
985 uint64_t IdxVal = foldImm(MO: IdxOperand, MRI: &MRI);
986 if (IdxVal < SrcTy.getNumElements()) {
987 LLT DstTy = MRI.getType(Reg: DstReg);
988 SmallVector<Register, 8> Regs;
989 SPIRVTypeInst DstSpvTy = GR->getSPIRVTypeForVReg(VReg: DstReg);
990 for (unsigned I = 0, E = SrcTy.getNumElements(); I < E; ++I) {
991 if (I == IdxVal) {
992 Regs.push_back(Elt: DstReg);
993 } else {
994 Register Reg = MRI.createGenericVirtualRegister(Ty: DstTy);
995 MRI.setRegClass(Reg, RC: GR->getRegClass(SpvType: DstSpvTy));
996 GR->assignSPIRVTypeToVReg(Type: DstSpvTy, VReg: Reg, MF: *MI.getMF());
997 Regs.push_back(Elt: Reg);
998 }
999 }
1000 MIRBuilder.buildUnmerge(Res: Regs, Op: SrcReg);
1001 MI.eraseFromParent();
1002 return true;
1003 }
1004 }
1005
1006 LLT EltTy = SrcTy.getElementType();
1007 Align VecAlign;
1008 MachinePointerInfo PtrInfo;
1009 auto StackTemp = createStackTemporaryForVector(Helper, GR, SrcReg, SrcTy,
1010 PtrInfo, VecAlign);
1011
1012 MIRBuilder.buildStore(Val: SrcReg, Addr: StackTemp, PtrInfo, Alignment: VecAlign);
1013
1014 Register IdxReg = IdxOperand.getReg();
1015 LLT PtrTy = MRI.getType(Reg: StackTemp.getReg(Idx: 0));
1016 Register EltPtr = MRI.createGenericVirtualRegister(Ty: PtrTy);
1017 auto Zero = MIRBuilder.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: 0);
1018
1019 MIRBuilder.buildIntrinsic(ID: Intrinsic::spv_gep, Res: ArrayRef<Register>{EltPtr})
1020 .addImm(Val: 1) // InBounds
1021 .addUse(RegNo: StackTemp.getReg(Idx: 0))
1022 .addUse(RegNo: Zero.getReg(Idx: 0))
1023 .addUse(RegNo: IdxReg);
1024
1025 MachinePointerInfo EltPtrInfo = MachinePointerInfo(PtrTy.getAddressSpace());
1026 Align EltAlign = Helper.getStackTemporaryAlignment(Type: EltTy);
1027 MIRBuilder.buildLoad(Res: DstReg, Addr: EltPtr, PtrInfo: EltPtrInfo, Alignment: EltAlign);
1028
1029 MI.eraseFromParent();
1030 return true;
1031 }
1032 return true;
1033}
1034
1035static bool legalizeSpvConstComposite(LegalizerHelper &Helper, MachineInstr &MI,
1036 SPIRVGlobalRegistry *GR) {
1037 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1038 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
1039 const SPIRVSubtarget &ST = MI.getMF()->getSubtarget<SPIRVSubtarget>();
1040
1041 Register DstReg = MI.getOperand(i: 0).getReg();
1042 LLT DstTy = MRI.getType(Reg: DstReg);
1043
1044 if (!needsVectorLegalization(Ty: DstTy, ST))
1045 return true;
1046
1047 SmallVector<Register, 8> SrcRegs;
1048 if (MI.getNumOperands() == 2) {
1049 // The "null" case: no values are attached.
1050 LLT EltTy = DstTy.getElementType();
1051 auto Zero = MIRBuilder.buildConstant(Res: EltTy, Val: 0);
1052 SPIRVTypeInst SpvDstTy = GR->getSPIRVTypeForVReg(VReg: DstReg);
1053 SPIRVTypeInst SpvEltTy = GR->getScalarOrVectorComponentType(Type: SpvDstTy);
1054 GR->assignSPIRVTypeToVReg(Type: SpvEltTy, VReg: Zero.getReg(Idx: 0), MF: MIRBuilder.getMF());
1055 for (unsigned i = 0; i < DstTy.getNumElements(); ++i)
1056 SrcRegs.push_back(Elt: Zero.getReg(Idx: 0));
1057 } else {
1058 for (unsigned i = 2; i < MI.getNumOperands(); ++i) {
1059 SrcRegs.push_back(Elt: MI.getOperand(i).getReg());
1060 }
1061 }
1062 MIRBuilder.buildBuildVector(Res: DstReg, Ops: SrcRegs);
1063 MI.eraseFromParent();
1064 return true;
1065}
1066
1067static SmallVector<Register, 16> unmergeToScalars(Register Reg,
1068 MachineIRBuilder &MIRBuilder,
1069 SPIRVGlobalRegistry *GR) {
1070 LLT Ty = MIRBuilder.getMRI()->getType(Reg);
1071 if (!Ty.isVector())
1072 return {Reg};
1073 SPIRVTypeInst EltSpvTy =
1074 GR->getScalarOrVectorComponentType(Type: GR->getSPIRVTypeForVReg(VReg: Reg));
1075 unsigned NumElts = Ty.getNumElements();
1076 SmallVector<Register, 16> Elts;
1077 for (unsigned I = 0; I < NumElts; ++I)
1078 Elts.push_back(Elt: createVirtualRegister(SpvType: EltSpvTy, GR, MIRBuilder));
1079 MIRBuilder.buildUnmerge(Res: Elts, Op: Reg);
1080 return Elts;
1081}
1082
1083static Register buildVectorPart(ArrayRef<Register> Elts,
1084 MachineIRBuilder &MIRBuilder,
1085 SPIRVGlobalRegistry *GR) {
1086 if (Elts.size() == 1)
1087 return Elts[0];
1088 SPIRVTypeInst PartSpvTy =
1089 GR->getOrCreateSPIRVVectorType(BaseType: GR->getSPIRVTypeForVReg(VReg: Elts[0]),
1090 NumElements: Elts.size(), MIRBuilder, /*EmitIR=*/true);
1091 Register Part = createVirtualRegister(SpvType: PartSpvTy, GR, MIRBuilder);
1092 MIRBuilder.buildBuildVector(Res: Part, Ops: Elts);
1093 return Part;
1094}
1095
1096// Split an elementwise intrinsic with an illegal vector width into intrinsics
1097// on legal vector widths.
1098static bool legalizeElementwiseIntrinsic(LegalizerHelper &Helper,
1099 GIntrinsic &MI,
1100 SPIRVGlobalRegistry *GR) {
1101 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1102 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
1103 const SPIRVSubtarget &ST = MI.getMF()->getSubtarget<SPIRVSubtarget>();
1104
1105 if (!Intrinsic::isTriviallyScalarizable(id: MI.getIntrinsicID()))
1106 return true;
1107 Register DstReg = MI.getReg(Idx: 0);
1108 LLT DstTy = MRI.getType(Reg: DstReg);
1109 if (!needsVectorLegalization(Ty: DstTy, ST))
1110 return true;
1111
1112 unsigned NumElts = DstTy.getNumElements();
1113 unsigned MaxVectorSize = ST.isShader() ? 4 : 16;
1114 unsigned PartSize = NumElts > MaxVectorSize ? MaxVectorSize : 4;
1115
1116 SmallDenseMap<Register, SmallVector<Register, 16>, 4> OpElts;
1117 for (const MachineOperand &MO : drop_begin(RangeOrContainer: MI.explicit_uses())) {
1118 if (!MO.isReg() || !MRI.getType(Reg: MO.getReg()).isVector())
1119 continue;
1120 auto [It, Inserted] = OpElts.try_emplace(Key: MO.getReg());
1121 if (Inserted)
1122 It->second = unmergeToScalars(Reg: MO.getReg(), MIRBuilder, GR);
1123 }
1124
1125 SPIRVTypeInst DstEltSpvTy =
1126 GR->getScalarOrVectorComponentType(Type: GR->getSPIRVTypeForVReg(VReg: DstReg));
1127 SmallVector<Register, 16> DstElts;
1128 for (unsigned Offset = 0; Offset < NumElts; Offset += PartSize) {
1129 unsigned Size = std::min(a: PartSize, b: NumElts - Offset);
1130 SPIRVTypeInst PartSpvTy =
1131 Size == 1 ? DstEltSpvTy
1132 : GR->getOrCreateSPIRVVectorType(BaseType: DstEltSpvTy, NumElements: Size,
1133 MIRBuilder, /*EmitIR=*/true);
1134 SmallDenseMap<Register, Register, 4> PartRegs;
1135 SmallVector<MachineOperand> PartOps;
1136 for (const MachineOperand &MO : drop_begin(RangeOrContainer: MI.explicit_uses())) {
1137 auto EltsIt = MO.isReg() ? OpElts.find(Val: MO.getReg()) : OpElts.end();
1138 if (EltsIt == OpElts.end()) {
1139 PartOps.push_back(Elt: MO);
1140 continue;
1141 }
1142 auto [It, Inserted] = PartRegs.try_emplace(Key: MO.getReg());
1143 if (Inserted)
1144 It->second = buildVectorPart(
1145 Elts: ArrayRef(EltsIt->second).slice(N: Offset, M: Size), MIRBuilder, GR);
1146 PartOps.push_back(Elt: MachineOperand::CreateReg(Reg: It->second, /*isDef=*/false));
1147 }
1148 Register PartDst = createVirtualRegister(SpvType: PartSpvTy, GR, MIRBuilder);
1149 auto Part = MIRBuilder.buildIntrinsic(
1150 ID: MI.getIntrinsicID(), Res: ArrayRef<Register>{PartDst}, HasSideEffects: MI.hasSideEffects(),
1151 isConvergent: MI.isConvergent());
1152 for (const MachineOperand &MO : PartOps)
1153 Part.add(MO);
1154 Part->setFlags(MI.getFlags());
1155 append_range(C&: DstElts, R: unmergeToScalars(Reg: PartDst, MIRBuilder, GR));
1156 }
1157
1158 MIRBuilder.buildBuildVector(Res: DstReg, Ops: DstElts);
1159 MI.eraseFromParent();
1160 return true;
1161}
1162
1163bool SPIRVLegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
1164 MachineInstr &MI) const {
1165 LLVM_DEBUG(dbgs() << "legalizeIntrinsic: " << MI);
1166 auto IntrinsicID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
1167 switch (IntrinsicID) {
1168 case Intrinsic::spv_bitcast:
1169 return legalizeSpvBitcast(Helper, MI, GR);
1170 case Intrinsic::spv_insertelt:
1171 return legalizeSpvInsertElt(Helper, MI, GR);
1172 case Intrinsic::spv_extractelt:
1173 return legalizeSpvExtractElt(Helper, MI, GR);
1174 case Intrinsic::spv_const_composite:
1175 return legalizeSpvConstComposite(Helper, MI, GR);
1176 }
1177 return legalizeElementwiseIntrinsic(Helper, MI&: cast<GIntrinsic>(Val&: MI), GR);
1178}
1179
1180bool SPIRVLegalizerInfo::legalizeBitcast(LegalizerHelper &Helper,
1181 MachineInstr &MI) const {
1182 // Once the G_BITCAST is using vectors that are allowed, we turn it back into
1183 // an spv_bitcast to avoid verifier problems when the register types are the
1184 // same for the source and the result. Note that the SPIR-V types associated
1185 // with the bitcast can be different even if the register types are the same.
1186 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1187 Register DstReg = MI.getOperand(i: 0).getReg();
1188 Register SrcReg = MI.getOperand(i: 1).getReg();
1189 SmallVector<Register, 1> DstRegs = {DstReg};
1190 MIRBuilder.buildIntrinsic(ID: Intrinsic::spv_bitcast, Res: DstRegs).addUse(RegNo: SrcReg);
1191 MI.eraseFromParent();
1192 return true;
1193}
1194
1195// Note this code was copied from LegalizerHelper::lowerISFPCLASS and adjusted
1196// to ensure that all instructions created during the lowering have SPIR-V types
1197// assigned to them.
1198bool SPIRVLegalizerInfo::legalizeIsFPClass(
1199 LegalizerHelper &Helper, MachineInstr &MI,
1200 LostDebugLocObserver &LocObserver) const {
1201 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
1202 FPClassTest Mask = static_cast<FPClassTest>(MI.getOperand(i: 2).getImm());
1203
1204 auto &MIRBuilder = Helper.MIRBuilder;
1205 auto &MF = MIRBuilder.getMF();
1206 MachineRegisterInfo &MRI = MF.getRegInfo();
1207
1208 Type *LLVMDstTy =
1209 IntegerType::get(C&: MIRBuilder.getContext(), NumBits: DstTy.getScalarSizeInBits());
1210 if (DstTy.isVector())
1211 LLVMDstTy = VectorType::get(ElementType: LLVMDstTy, EC: DstTy.getElementCount());
1212 SPIRVTypeInst SPIRVDstTy = GR->getOrCreateSPIRVType(
1213 Type: LLVMDstTy, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite,
1214 /*EmitIR*/ true);
1215
1216 unsigned BitSize = SrcTy.getScalarSizeInBits();
1217 const fltSemantics &Semantics = getFltSemanticForLLT(Ty: SrcTy.getScalarType());
1218
1219 LLT IntTy = LLT::scalar(SizeInBits: BitSize);
1220 Type *LLVMIntTy = IntegerType::get(C&: MIRBuilder.getContext(), NumBits: BitSize);
1221 if (SrcTy.isVector()) {
1222 IntTy = LLT::vector(EC: SrcTy.getElementCount(), ScalarTy: IntTy);
1223 LLVMIntTy = VectorType::get(ElementType: LLVMIntTy, EC: SrcTy.getElementCount());
1224 }
1225 SPIRVTypeInst SPIRVIntTy = GR->getOrCreateSPIRVType(
1226 Type: LLVMIntTy, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite,
1227 /*EmitIR*/ true);
1228
1229 // Clang doesn't support capture of structured bindings:
1230 LLT DstTyCopy = DstTy;
1231 const auto assignSPIRVTy = [&](MachineInstrBuilder &&MI) {
1232 // Assign this MI's (assumed only) destination to one of the two types we
1233 // expect: either the G_IS_FPCLASS's destination type, or the integer type
1234 // bitcast from the source type.
1235 LLT MITy = MRI.getType(Reg: MI.getReg(Idx: 0));
1236 assert((MITy == IntTy || MITy == DstTyCopy) &&
1237 "Unexpected LLT type while lowering G_IS_FPCLASS");
1238 SPIRVTypeInst SPVTy = MITy == IntTy ? SPIRVIntTy : SPIRVDstTy;
1239 GR->assignSPIRVTypeToVReg(Type: SPVTy, VReg: MI.getReg(Idx: 0), MF);
1240 return MI;
1241 };
1242
1243 // Helper to build and assign a constant in one go
1244 const auto buildSPIRVConstant = [&](LLT Ty, auto &&C) -> MachineInstrBuilder {
1245 if (!Ty.isFixedVector())
1246 return assignSPIRVTy(MIRBuilder.buildConstant(Ty, C));
1247 auto ScalarC = MIRBuilder.buildConstant(Ty.getScalarType(), C);
1248 assert((Ty == IntTy || Ty == DstTyCopy) &&
1249 "Unexpected LLT type while lowering constant for G_IS_FPCLASS");
1250 SPIRVTypeInst VecEltTy = GR->getOrCreateSPIRVType(
1251 Type: (Ty == IntTy ? LLVMIntTy : LLVMDstTy)->getScalarType(), MIRBuilder,
1252 AQ: SPIRV::AccessQualifier::ReadWrite,
1253 /*EmitIR*/ true);
1254 GR->assignSPIRVTypeToVReg(Type: VecEltTy, VReg: ScalarC.getReg(0), MF);
1255 return assignSPIRVTy(MIRBuilder.buildSplatBuildVector(Res: Ty, Src: ScalarC));
1256 };
1257
1258 if (Mask == fcNone) {
1259 MIRBuilder.buildCopy(Res: DstReg, Op: buildSPIRVConstant(DstTy, 0));
1260 MI.eraseFromParent();
1261 return true;
1262 }
1263 if (Mask == fcAllFlags) {
1264 MIRBuilder.buildCopy(Res: DstReg, Op: buildSPIRVConstant(DstTy, 1));
1265 MI.eraseFromParent();
1266 return true;
1267 }
1268
1269 // Note that rather than creating a COPY here (between a floating-point and
1270 // integer type of the same size) we create a SPIR-V bitcast immediately. We
1271 // can't create a G_BITCAST because the LLTs are the same, and we can't seem
1272 // to correctly lower COPYs to SPIR-V bitcasts at this moment.
1273 Register ResVReg = MRI.createGenericVirtualRegister(Ty: IntTy);
1274 MRI.setRegClass(Reg: ResVReg, RC: GR->getRegClass(SpvType: SPIRVIntTy));
1275 GR->assignSPIRVTypeToVReg(Type: SPIRVIntTy, VReg: ResVReg, MF: Helper.MIRBuilder.getMF());
1276 auto AsInt = MIRBuilder.buildInstr(Opcode: SPIRV::OpBitcast)
1277 .addDef(RegNo: ResVReg)
1278 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: SPIRVIntTy))
1279 .addUse(RegNo: SrcReg);
1280 AsInt = assignSPIRVTy(std::move(AsInt));
1281
1282 // Various masks.
1283 APInt SignBit = APInt::getSignMask(BitWidth: BitSize);
1284 APInt ValueMask = APInt::getSignedMaxValue(numBits: BitSize); // All bits but sign.
1285 APInt Inf = APFloat::getInf(Sem: Semantics).bitcastToAPInt(); // Exp and int bit.
1286 APInt ExpMask = Inf;
1287 APInt AllOneMantissa = APFloat::getLargest(Sem: Semantics).bitcastToAPInt() & ~Inf;
1288 APInt QNaNBitMask =
1289 APInt::getOneBitSet(numBits: BitSize, BitNo: AllOneMantissa.getActiveBits() - 1);
1290 APInt InversionMask = APInt::getAllOnes(numBits: DstTy.getScalarSizeInBits());
1291
1292 auto SignBitC = buildSPIRVConstant(IntTy, SignBit);
1293 auto ValueMaskC = buildSPIRVConstant(IntTy, ValueMask);
1294 auto InfC = buildSPIRVConstant(IntTy, Inf);
1295 auto ExpMaskC = buildSPIRVConstant(IntTy, ExpMask);
1296 auto ZeroC = buildSPIRVConstant(IntTy, 0);
1297
1298 auto Abs = assignSPIRVTy(MIRBuilder.buildAnd(Dst: IntTy, Src0: AsInt, Src1: ValueMaskC));
1299 auto Sign = assignSPIRVTy(
1300 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_NE, Res: DstTy, Op0: AsInt, Op1: Abs));
1301
1302 auto Res = buildSPIRVConstant(DstTy, 0);
1303
1304 const auto appendToRes = [&](MachineInstrBuilder &&ToAppend) {
1305 Res = assignSPIRVTy(
1306 MIRBuilder.buildOr(Dst: DstTyCopy, Src0: Res, Src1: assignSPIRVTy(std::move(ToAppend))));
1307 };
1308
1309 // Tests that involve more than one class should be processed first.
1310 if ((Mask & fcFinite) == fcFinite) {
1311 // finite(V) ==> abs(V) u< exp_mask
1312 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: Abs,
1313 Op1: ExpMaskC));
1314 Mask &= ~fcFinite;
1315 } else if ((Mask & fcFinite) == fcPosFinite) {
1316 // finite(V) && V > 0 ==> V u< exp_mask
1317 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: AsInt,
1318 Op1: ExpMaskC));
1319 Mask &= ~fcPosFinite;
1320 } else if ((Mask & fcFinite) == fcNegFinite) {
1321 // finite(V) && V < 0 ==> abs(V) u< exp_mask && signbit == 1
1322 auto Cmp = assignSPIRVTy(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT,
1323 Res: DstTy, Op0: Abs, Op1: ExpMaskC));
1324 appendToRes(MIRBuilder.buildAnd(Dst: DstTy, Src0: Cmp, Src1: Sign));
1325 Mask &= ~fcNegFinite;
1326 }
1327
1328 if (FPClassTest PartialCheck = Mask & (fcZero | fcSubnormal)) {
1329 // fcZero | fcSubnormal => test all exponent bits are 0
1330 // TODO: Handle sign bit specific cases
1331 // TODO: Handle inverted case
1332 if (PartialCheck == (fcZero | fcSubnormal)) {
1333 auto ExpBits = assignSPIRVTy(MIRBuilder.buildAnd(Dst: IntTy, Src0: AsInt, Src1: ExpMaskC));
1334 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
1335 Op0: ExpBits, Op1: ZeroC));
1336 Mask &= ~PartialCheck;
1337 }
1338 }
1339
1340 // Check for individual classes.
1341 if (FPClassTest PartialCheck = Mask & fcZero) {
1342 if (PartialCheck == fcPosZero)
1343 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
1344 Op0: AsInt, Op1: ZeroC));
1345 else if (PartialCheck == fcZero)
1346 appendToRes(
1347 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy, Op0: Abs, Op1: ZeroC));
1348 else // fcNegZero
1349 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
1350 Op0: AsInt, Op1: SignBitC));
1351 }
1352
1353 if (FPClassTest PartialCheck = Mask & fcSubnormal) {
1354 // issubnormal(V) ==> unsigned(abs(V) - 1) u< (all mantissa bits set)
1355 // issubnormal(V) && V>0 ==> unsigned(V - 1) u< (all mantissa bits set)
1356 auto V = (PartialCheck == fcPosSubnormal) ? AsInt : Abs;
1357 auto OneC = buildSPIRVConstant(IntTy, 1);
1358 auto VMinusOne = MIRBuilder.buildSub(Dst: IntTy, Src0: V, Src1: OneC);
1359 auto SubnormalRes = assignSPIRVTy(
1360 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: VMinusOne,
1361 Op1: buildSPIRVConstant(IntTy, AllOneMantissa)));
1362 if (PartialCheck == fcNegSubnormal)
1363 SubnormalRes = MIRBuilder.buildAnd(Dst: DstTy, Src0: SubnormalRes, Src1: Sign);
1364 appendToRes(std::move(SubnormalRes));
1365 }
1366
1367 if (FPClassTest PartialCheck = Mask & fcInf) {
1368 if (PartialCheck == fcPosInf)
1369 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
1370 Op0: AsInt, Op1: InfC));
1371 else if (PartialCheck == fcInf)
1372 appendToRes(
1373 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy, Op0: Abs, Op1: InfC));
1374 else { // fcNegInf
1375 APInt NegInf = APFloat::getInf(Sem: Semantics, Negative: true).bitcastToAPInt();
1376 auto NegInfC = buildSPIRVConstant(IntTy, NegInf);
1377 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
1378 Op0: AsInt, Op1: NegInfC));
1379 }
1380 }
1381
1382 if (FPClassTest PartialCheck = Mask & fcNan) {
1383 auto InfWithQnanBitC =
1384 buildSPIRVConstant(IntTy, std::move(Inf) | QNaNBitMask);
1385 if (PartialCheck == fcNan) {
1386 // isnan(V) ==> abs(V) u> int(inf)
1387 appendToRes(
1388 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_UGT, Res: DstTy, Op0: Abs, Op1: InfC));
1389 } else if (PartialCheck == fcQNan) {
1390 // isquiet(V) ==> abs(V) u>= (unsigned(Inf) | quiet_bit)
1391 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_UGE, Res: DstTy, Op0: Abs,
1392 Op1: InfWithQnanBitC));
1393 } else { // fcSNan
1394 // issignaling(V) ==> abs(V) u> unsigned(Inf) &&
1395 // abs(V) u< (unsigned(Inf) | quiet_bit)
1396 auto IsNan = assignSPIRVTy(
1397 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_UGT, Res: DstTy, Op0: Abs, Op1: InfC));
1398 auto IsNotQnan = assignSPIRVTy(MIRBuilder.buildICmp(
1399 Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: Abs, Op1: InfWithQnanBitC));
1400 appendToRes(MIRBuilder.buildAnd(Dst: DstTy, Src0: IsNan, Src1: IsNotQnan));
1401 }
1402 }
1403
1404 if (FPClassTest PartialCheck = Mask & fcNormal) {
1405 // isnormal(V) ==> (0 u< exp u< max_exp) ==> (unsigned(exp-1) u<
1406 // (max_exp-1))
1407 APInt ExpLSB = ExpMask & ~(ExpMask.shl(shiftAmt: 1));
1408 auto ExpMinusOne = assignSPIRVTy(
1409 MIRBuilder.buildSub(Dst: IntTy, Src0: Abs, Src1: buildSPIRVConstant(IntTy, ExpLSB)));
1410 APInt MaxExpMinusOne = std::move(ExpMask) - ExpLSB;
1411 auto NormalRes = assignSPIRVTy(
1412 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: ExpMinusOne,
1413 Op1: buildSPIRVConstant(IntTy, MaxExpMinusOne)));
1414 if (PartialCheck == fcNegNormal)
1415 NormalRes = MIRBuilder.buildAnd(Dst: DstTy, Src0: NormalRes, Src1: Sign);
1416 else if (PartialCheck == fcPosNormal) {
1417 auto PosSign = assignSPIRVTy(MIRBuilder.buildXor(
1418 Dst: DstTy, Src0: Sign, Src1: buildSPIRVConstant(DstTy, InversionMask)));
1419 NormalRes = MIRBuilder.buildAnd(Dst: DstTy, Src0: NormalRes, Src1: PosSign);
1420 }
1421 appendToRes(std::move(NormalRes));
1422 }
1423
1424 MIRBuilder.buildCopy(Res: DstReg, Op: Res);
1425 MI.eraseFromParent();
1426 return true;
1427}
1428