1//===- SPIRVLegalizerInfo.cpp --- SPIR-V Legalization Rules ------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the targeting of the Machinelegalizer class for SPIR-V.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SPIRVLegalizerInfo.h"
14#include "SPIRV.h"
15#include "SPIRVGlobalRegistry.h"
16#include "SPIRVSubtarget.h"
17#include "SPIRVUtils.h"
18#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
19#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
20#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
21#include "llvm/CodeGen/MachineInstr.h"
22#include "llvm/CodeGen/MachineRegisterInfo.h"
23#include "llvm/CodeGen/TargetOpcodes.h"
24#include "llvm/IR/IntrinsicsSPIRV.h"
25#include "llvm/Support/Debug.h"
26#include "llvm/Support/MathExtras.h"
27
28using namespace llvm;
29using namespace llvm::LegalizeActions;
30using namespace llvm::LegalityPredicates;
31
32#define DEBUG_TYPE "spirv-legalizer"
33
34LegalityPredicate typeOfExtendedScalars(unsigned TypeIdx, bool IsExtendedInts) {
35 return [IsExtendedInts, TypeIdx](const LegalityQuery &Query) {
36 const LLT Ty = Query.Types[TypeIdx];
37 return IsExtendedInts && Ty.isValid() && Ty.isScalar();
38 };
39}
40
41LegalityPredicate typeOfLongVectors(unsigned TypeIdx, bool IsLongVecs) {
42 return [TypeIdx, IsLongVecs](const LegalityQuery &Query) {
43 const LLT Ty = Query.Types[TypeIdx];
44 return IsLongVecs && Ty.isValid() && Ty.isVector();
45 };
46}
47
48SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) {
49 using namespace TargetOpcode;
50
51 this->ST = &ST;
52 GR = ST.getSPIRVGlobalRegistry();
53
54 const LLT s1 = LLT::scalar(SizeInBits: 1);
55 const LLT s8 = LLT::scalar(SizeInBits: 8);
56 const LLT s16 = LLT::scalar(SizeInBits: 16);
57 const LLT s32 = LLT::scalar(SizeInBits: 32);
58 const LLT s64 = LLT::scalar(SizeInBits: 64);
59 const LLT s128 = LLT::scalar(SizeInBits: 128);
60
61 const LLT v16s64 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 64);
62 const LLT v16s32 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 32);
63 const LLT v16s16 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 16);
64 const LLT v16s8 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 8);
65 const LLT v16s1 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 1);
66
67 const LLT v8s64 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 64);
68 const LLT v8s32 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 32);
69 const LLT v8s16 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16);
70 const LLT v8s8 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 8);
71 const LLT v8s1 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 1);
72
73 const LLT v4s64 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 64);
74 const LLT v4s32 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32);
75 const LLT v4s16 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16);
76 const LLT v4s8 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 8);
77 const LLT v4s1 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 1);
78
79 const LLT v3s64 = LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 64);
80 const LLT v3s32 = LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 32);
81 const LLT v3s16 = LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 16);
82 const LLT v3s8 = LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 8);
83 const LLT v3s1 = LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 1);
84
85 const LLT v2s64 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64);
86 const LLT v2s32 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32);
87 const LLT v2s16 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16);
88 const LLT v2s8 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 8);
89 const LLT v2s1 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 1);
90
91 const unsigned PSize = ST.getPointerSize();
92 const LLT p0 = LLT::pointer(AddressSpace: 0, SizeInBits: PSize); // Function
93 const LLT p1 = LLT::pointer(AddressSpace: 1, SizeInBits: PSize); // CrossWorkgroup
94 const LLT p2 = LLT::pointer(AddressSpace: 2, SizeInBits: PSize); // UniformConstant
95 const LLT p3 = LLT::pointer(AddressSpace: 3, SizeInBits: PSize); // Workgroup
96 const LLT p4 = LLT::pointer(AddressSpace: 4, SizeInBits: PSize); // Generic
97 const LLT p5 =
98 LLT::pointer(AddressSpace: 5, SizeInBits: PSize); // Input, SPV_INTEL_usm_storage_classes (Device)
99 const LLT p6 = LLT::pointer(AddressSpace: 6, SizeInBits: PSize); // SPV_INTEL_usm_storage_classes (Host)
100 const LLT p7 = LLT::pointer(AddressSpace: 7, SizeInBits: PSize); // Input
101 const LLT p8 = LLT::pointer(AddressSpace: 8, SizeInBits: PSize); // Output
102 const LLT p9 =
103 LLT::pointer(AddressSpace: 9, SizeInBits: PSize); // CodeSectionINTEL, SPV_INTEL_function_pointers
104 const LLT p10 = LLT::pointer(AddressSpace: 10, SizeInBits: PSize); // Private
105 const LLT p11 = LLT::pointer(AddressSpace: 11, SizeInBits: PSize); // StorageBuffer
106 const LLT p12 = LLT::pointer(AddressSpace: 12, SizeInBits: PSize); // Uniform
107 const LLT p13 = LLT::pointer(AddressSpace: 13, SizeInBits: PSize); // PushConstant
108
109 // TODO: remove copy-pasting here by using concatenation in some way.
110 auto allPtrsScalarsAndVectors = {
111 p0, p1, p2, p3, p4, p5, p6, p7, p8,
112 p9, p10, p11, p12, p13, s1, s8, s16, s32,
113 s64, s128, v2s1, v2s8, v2s16, v2s32, v2s64, v3s1, v3s8,
114 v3s16, v3s32, v3s64, v4s1, v4s8, v4s16, v4s32, v4s64, v8s1,
115 v8s8, v8s16, v8s32, v8s64, v16s1, v16s8, v16s16, v16s32, v16s64};
116
117 auto allVectors = {v2s1, v2s8, v2s16, v2s32, v2s64, v3s1, v3s8,
118 v3s16, v3s32, v3s64, v4s1, v4s8, v4s16, v4s32,
119 v4s64, v8s1, v8s8, v8s16, v8s32, v8s64, v16s1,
120 v16s8, v16s16, v16s32, v16s64};
121
122 auto allShaderVectors = {v2s1, v2s8, v2s16, v2s32, v2s64,
123 v3s1, v3s8, v3s16, v3s32, v3s64,
124 v4s1, v4s8, v4s16, v4s32, v4s64};
125
126 auto allScalars = {s1, s8, s16, s32, s64};
127
128 auto allScalarsAndVectors = {
129 s1, s8, s16, s32, s64, s128, v2s1, v2s8,
130 v2s16, v2s32, v2s64, v3s1, v3s8, v3s16, v3s32, v3s64,
131 v4s1, v4s8, v4s16, v4s32, v4s64, v8s1, v8s8, v8s16,
132 v8s32, v8s64, v16s1, v16s8, v16s16, v16s32, v16s64};
133
134 auto allShaderScalarsAndVectors = {
135 s1, s8, s16, s32, s64, s128, v2s1, v2s8, v2s16, v2s32, v2s64,
136 v3s1, v3s8, v3s16, v3s32, v3s64, v4s1, v4s8, v4s16, v4s32, v4s64};
137
138 auto &allowedScalarsAndVectors =
139 ST.isShader() ? allShaderScalarsAndVectors : allScalarsAndVectors;
140
141 auto allIntScalarsAndVectors = {
142 s8, s16, s32, s64, s128, v2s8, v2s16, v2s32, v2s64,
143 v3s8, v3s16, v3s32, v3s64, v4s8, v4s16, v4s32, v4s64, v8s8,
144 v8s16, v8s32, v8s64, v16s8, v16s16, v16s32, v16s64};
145
146 auto allBoolScalarsAndVectors = {s1, v2s1, v3s1, v4s1, v8s1, v16s1};
147 auto allBoolVectors = {v2s1, v3s1, v4s1, v8s1, v16s1};
148
149 auto allIntScalars = {s8, s16, s32, s64, s128};
150
151 auto allShaderIntVectors = {v2s8, v2s16, v2s32, v2s64, v3s8, v3s16,
152 v3s32, v3s64, v4s8, v4s16, v4s32, v4s64};
153
154 auto allIntVectors = {v2s8, v2s16, v2s32, v2s64, v3s8, v3s16, v3s32,
155 v3s64, v4s8, v4s16, v4s32, v4s64, v8s8, v8s16,
156 v8s32, v8s64, v16s8, v16s16, v16s32, v16s64};
157
158 auto &allowedIntVectorTypes =
159 ST.isShader() ? allShaderIntVectors : allIntVectors;
160
161 auto allFloatScalarsAndF16Vector2AndVector4s = {s16, s32, s64, v2s16, v4s16};
162
163 auto allFloatScalars = {s16, s32, s64};
164
165 auto allFloatScalarsAndVectors = {
166 s16, s32, s64, v2s16, v2s32, v2s64, v3s16, v3s32, v3s64,
167 v4s16, v4s32, v4s64, v8s16, v8s32, v8s64, v16s16, v16s32, v16s64};
168
169 auto allShaderFloatVectors = {v2s16, v2s32, v2s64, v3s16, v3s32,
170 v3s64, v4s16, v4s32, v4s64};
171
172 auto allFloatVectors = {v2s16, v2s32, v2s64, v3s16, v3s32,
173 v3s64, v4s16, v4s32, v4s64, v8s16,
174 v8s32, v8s64, v16s16, v16s32, v16s64};
175
176 auto &allowedFloatVectorTypes =
177 ST.isShader() ? allShaderFloatVectors : allFloatVectors;
178
179 auto allFloatAndIntScalarsAndPtrs = {s8, s16, s32, s64, p0, p1,
180 p2, p3, p4, p5, p6, p7,
181 p8, p9, p10, p11, p12, p13};
182
183 auto allPtrs = {p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13};
184
185 auto &allowedVectorTypes = ST.isShader() ? allShaderVectors : allVectors;
186
187 bool HasArbitraryPrecisionInts = ST.canUseExtension(
188 E: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers);
189 bool IsExtendedInts =
190 HasArbitraryPrecisionInts ||
191 ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_bit_instructions) ||
192 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_int4);
193 bool IsLongVecs = ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_long_vector);
194 auto ExtendedIntScalarsAndVectors =
195 [IsExtendedInts](const LegalityQuery &Query) {
196 const LLT Ty = Query.Types[0];
197 return IsExtendedInts && Ty.isValid() &&
198 !Ty.isPointerOrPointerVector() && Ty.getScalarSizeInBits() > 1;
199 };
200 auto ExtendedScalarsAndVectorsProduct = [IsExtendedInts](
201 const LegalityQuery &Query) {
202 const LLT Ty1 = Query.Types[0], Ty2 = Query.Types[1];
203 return IsExtendedInts && Ty1.isValid() && Ty2.isValid() &&
204 !Ty1.isPointerOrPointerVector() && !Ty2.isPointerOrPointerVector();
205 };
206 auto ExtendedPtrsScalarsAndVectors =
207 [IsExtendedInts](const LegalityQuery &Query) {
208 const LLT Ty = Query.Types[0];
209 return IsExtendedInts && Ty.isValid();
210 };
211
212 // The universal validation rules in the SPIR-V specification state that
213 // vector sizes are typically limited to 2, 3, or 4. However, larger vector
214 // sizes (8 and 16) are enabled when the Kernel capability is present. For
215 // shader execution models, vector sizes are strictly limited to 4. In
216 // non-shader contexts, vector sizes of 8 and 16 are also permitted, but
217 // arbitrary sizes (e.g., 6 or 11) are not.
218 uint32_t MaxVectorSize = ST.isShader() ? 4 : 16;
219 LLVM_DEBUG(dbgs() << "MaxVectorSize: " << MaxVectorSize << "\n");
220
221 for (auto Opc : getTypeFoldingSupportedOpcodes()) {
222 switch (Opc) {
223 case G_EXTRACT_VECTOR_ELT:
224 case G_UREM:
225 case G_SREM:
226 case G_UDIV:
227 case G_SDIV:
228 case G_FREM:
229 break;
230 default:
231 getActionDefinitionsBuilder(Opcode: Opc)
232 .customFor(Types: allScalars)
233 .customFor(Types: allowedVectorTypes)
234 .customIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
235 .moreElementsToNextPow2(TypeIdx: 0)
236 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
237 Mutation: LegalizeMutations::changeElementCountTo(
238 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
239 .custom();
240 break;
241 }
242 }
243
244 getActionDefinitionsBuilder(Opcodes: {G_UREM, G_SREM, G_SDIV, G_UDIV, G_FREM})
245 .customFor(Types: allScalars)
246 .customFor(Types: allowedVectorTypes)
247 .customIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
248 .scalarizeIf(Predicate: numElementsNotPow2(TypeIdx: 0), TypeIdx: 0)
249 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
250 Mutation: LegalizeMutations::changeElementCountTo(
251 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
252 .custom();
253
254 getActionDefinitionsBuilder(Opcodes: {G_FMA, G_STRICT_FMA})
255 .legalFor(Types: allScalars)
256 .legalFor(Types: allowedVectorTypes)
257 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
258 .moreElementsToNextPow2(TypeIdx: 0)
259 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
260 Mutation: LegalizeMutations::changeElementCountTo(
261 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
262 .alwaysLegal();
263
264 getActionDefinitionsBuilder(Opcode: G_INTRINSIC_W_SIDE_EFFECTS).custom();
265
266 getActionDefinitionsBuilder(Opcode: G_SHUFFLE_VECTOR)
267 .legalForCartesianProduct(Types0: allowedVectorTypes, Types1: allowedVectorTypes)
268 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
269 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 1, IsLongVecs))
270 .moreElementsToNextPow2(TypeIdx: 0)
271 .lowerIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize))
272 .moreElementsToNextPow2(TypeIdx: 1)
273 .lowerIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 1, Size: MaxVectorSize));
274
275 getActionDefinitionsBuilder(Opcode: G_EXTRACT_VECTOR_ELT)
276 .customIf(Predicate: typeOfLongVectors(TypeIdx: 1, IsLongVecs))
277 .moreElementsToNextPow2(TypeIdx: 1)
278 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 1, Size: MaxVectorSize),
279 Mutation: LegalizeMutations::changeElementCountTo(
280 TypeIdx: 1, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
281 .custom();
282
283 getActionDefinitionsBuilder(Opcode: G_INSERT_VECTOR_ELT)
284 .customIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
285 .moreElementsToNextPow2(TypeIdx: 0)
286 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
287 Mutation: LegalizeMutations::changeElementCountTo(
288 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
289 .custom();
290
291 // Illegal G_UNMERGE_VALUES instructions should be handled
292 // during the combine phase.
293 getActionDefinitionsBuilder(Opcode: G_BUILD_VECTOR)
294 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
295 .legalIf(Predicate: vectorElementCountIsLessThanOrEqualTo(TypeIdx: 0, Size: MaxVectorSize))
296 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
297 Mutation: LegalizeMutations::changeElementCountTo(
298 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)));
299
300 // When entering the legalizer, there should be no G_BITCAST instructions.
301 // They should all be calls to the `spv_bitcast` intrinsic. The call to
302 // the intrinsic will be converted to a G_BITCAST during legalization if
303 // the vectors are not legal. After using the rules to legalize a G_BITCAST,
304 // we turn it back into a call to the intrinsic with a custom rule to avoid
305 // potential machine verifier failures.
306 getActionDefinitionsBuilder(Opcode: G_BITCAST)
307 .customIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
308 .moreElementsToNextPow2(TypeIdx: 0)
309 .moreElementsToNextPow2(TypeIdx: 1)
310 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
311 Mutation: LegalizeMutations::changeElementCountTo(
312 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
313 .lowerIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 1, Size: MaxVectorSize))
314 .custom();
315
316 // If the result is still illegal, the combiner should be able to remove it.
317 getActionDefinitionsBuilder(Opcode: G_CONCAT_VECTORS)
318 .legalForCartesianProduct(Types0: allowedVectorTypes, Types1: allowedVectorTypes)
319 .legalIf(Predicate: LegalityPredicates::any(P0: typeOfLongVectors(TypeIdx: 0, IsLongVecs),
320 P1: typeOfLongVectors(TypeIdx: 1, IsLongVecs)));
321
322 getActionDefinitionsBuilder(Opcode: G_SPLAT_VECTOR)
323 .legalFor(Types: allowedVectorTypes)
324 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
325 .moreElementsToNextPow2(TypeIdx: 0)
326 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
327 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx: 0, FromTypeIdx: MaxVectorSize))
328 .alwaysLegal();
329
330 // Vector Reduction Operations
331 getActionDefinitionsBuilder(
332 Opcodes: {G_VECREDUCE_SMIN, G_VECREDUCE_SMAX, G_VECREDUCE_UMIN, G_VECREDUCE_UMAX,
333 G_VECREDUCE_ADD, G_VECREDUCE_MUL, G_VECREDUCE_FMUL, G_VECREDUCE_FMIN,
334 G_VECREDUCE_FMAX, G_VECREDUCE_FMINIMUM, G_VECREDUCE_FMAXIMUM,
335 G_VECREDUCE_OR, G_VECREDUCE_AND, G_VECREDUCE_XOR})
336 .legalFor(Types: allowedVectorTypes)
337 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
338 .scalarize(TypeIdx: 1)
339 .lower();
340
341 getActionDefinitionsBuilder(Opcodes: {G_VECREDUCE_SEQ_FADD, G_VECREDUCE_SEQ_FMUL})
342 .scalarize(TypeIdx: 2)
343 .lower();
344
345 // Illegal G_UNMERGE_VALUES instructions should be handled
346 // during the combine phase.
347 getActionDefinitionsBuilder(Opcode: G_UNMERGE_VALUES)
348 .legalIf(Predicate: LegalityPredicates::any(P0: typeOfLongVectors(TypeIdx: 0, IsLongVecs),
349 P1: typeOfLongVectors(TypeIdx: 1, IsLongVecs)))
350 .legalIf(Predicate: vectorElementCountIsLessThanOrEqualTo(TypeIdx: 1, Size: MaxVectorSize));
351
352 getActionDefinitionsBuilder(Opcodes: {G_MEMCPY, G_MEMCPY_INLINE, G_MEMMOVE})
353 .unsupportedIf(Predicate: LegalityPredicates::any(P0: typeIs(TypeIdx: 0, TypesInit: p9), P1: typeIs(TypeIdx: 1, TypesInit: p9)))
354 .legalIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allPtrs), P1: typeInSet(TypeIdx: 1, TypesInit: allPtrs)));
355
356 getActionDefinitionsBuilder(Opcodes: {G_MEMSET, G_MEMSET_INLINE})
357 .unsupportedIf(Predicate: typeIs(TypeIdx: 0, TypesInit: p9))
358 .legalIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allPtrs), P1: typeInSet(TypeIdx: 1, TypesInit: allIntScalars)));
359
360 getActionDefinitionsBuilder(Opcode: G_ADDRSPACE_CAST)
361 .legalForCartesianProduct(Types0: allPtrs, Types1: allPtrs);
362
363 // Should we be legalizing bad scalar sizes like s5 here instead
364 // of handling them in the instruction selector?
365 getActionDefinitionsBuilder(Opcodes: {G_LOAD, G_STORE})
366 .unsupportedIf(Predicate: typeIs(TypeIdx: 1, TypesInit: p9))
367 .legalForCartesianProduct(Types0: allowedVectorTypes, Types1: allPtrs)
368 .legalForCartesianProduct(Types0: allPtrs, Types1: allPtrs)
369 .legalIf(Predicate: isScalar(TypeIdx: 0))
370 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
371 .custom();
372
373 getActionDefinitionsBuilder(Opcodes: {G_SMIN, G_SMAX, G_UMIN, G_UMAX, G_ABS,
374 G_BITREVERSE, G_SADDSAT, G_UADDSAT, G_SSUBSAT,
375 G_USUBSAT, G_SCMP, G_UCMP})
376 .legalFor(Types: allIntScalars)
377 .legalFor(Types: allowedIntVectorTypes)
378 .legalIf(Predicate: ExtendedIntScalarsAndVectors)
379 // LLVM i1 maps to OpTypeBool, not OpTypeInt.
380 .scalarizeIf(Predicate: typeInSet(TypeIdx: 0, TypesInit: allBoolVectors), TypeIdx: 0)
381 .minScalar(TypeIdx: 0, Ty: s32)
382 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
383 Mutation: LegalizeMutations::changeElementCountTo(
384 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
385 .moreElementsToNextPow2(TypeIdx: 0);
386
387 getActionDefinitionsBuilder(Opcodes: {G_SSHLSAT, G_USHLSAT}).lower();
388
389 getActionDefinitionsBuilder(Opcodes: {G_FLDEXP, G_STRICT_FLDEXP})
390 .legalForCartesianProduct(Types0: allFloatScalarsAndVectors, Types1: allIntScalars);
391
392 getActionDefinitionsBuilder(Opcodes: {G_FPTOSI, G_FPTOUI})
393 .legalForCartesianProduct(Types0: allIntScalarsAndVectors,
394 Types1: allFloatScalarsAndVectors);
395
396 getActionDefinitionsBuilder(Opcodes: {G_FPTOSI_SAT, G_FPTOUI_SAT})
397 .legalForCartesianProduct(Types0: allIntScalarsAndVectors,
398 Types1: allFloatScalarsAndVectors);
399
400 getActionDefinitionsBuilder(Opcodes: {G_SITOFP, G_UITOFP})
401 .legalForCartesianProduct(Types0: allFloatScalarsAndVectors,
402 Types1: allScalarsAndVectors);
403
404 getActionDefinitionsBuilder(Opcode: G_CTPOP)
405 .legalForCartesianProduct(Types: allIntScalarsAndVectors)
406 .legalIf(Predicate: ExtendedScalarsAndVectorsProduct)
407 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs));
408
409 getActionDefinitionsBuilder(Opcodes: {G_TRUNC, G_ZEXT, G_SEXT, G_ANYEXT})
410 .legalForCartesianProduct(Types: allowedScalarsAndVectors)
411 .legalIf(Predicate: ExtendedScalarsAndVectorsProduct)
412 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
413 .moreElementsToNextPow2(TypeIdx: 0)
414 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
415 Mutation: LegalizeMutations::changeElementCountTo(
416 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)));
417
418 getActionDefinitionsBuilder(Opcode: G_SEXT_INREG)
419 .lowerIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
420 .moreElementsToNextPow2(TypeIdx: 0)
421 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
422 Mutation: LegalizeMutations::changeElementCountTo(
423 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
424 .lower();
425
426 getActionDefinitionsBuilder(Opcode: G_PHI)
427 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
428 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
429 Mutation: LegalizeMutations::changeElementCountTo(
430 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
431 .legalFor(Types: allPtrsScalarsAndVectors)
432 .legalIf(Predicate: ExtendedPtrsScalarsAndVectors)
433 .moreElementsToNextPow2(TypeIdx: 0);
434
435 getActionDefinitionsBuilder(Opcode: G_BITCAST).legalIf(
436 Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allPtrsScalarsAndVectors),
437 P1: typeInSet(TypeIdx: 1, TypesInit: allPtrsScalarsAndVectors)));
438
439 getActionDefinitionsBuilder(Opcodes: {G_IMPLICIT_DEF, G_FREEZE})
440 .legalFor(Types: {s1, s128})
441 .legalFor(Types: allFloatAndIntScalarsAndPtrs)
442 .legalFor(Types: allowedVectorTypes)
443 .legalIf(Predicate: [](const LegalityQuery &Query) {
444 return Query.Types[0].isPointerVector();
445 })
446 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
447 .moreElementsToNextPow2(TypeIdx: 0)
448 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
449 Mutation: LegalizeMutations::changeElementCountTo(
450 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)));
451
452 getActionDefinitionsBuilder(Opcodes: {G_STACKSAVE, G_STACKRESTORE}).alwaysLegal();
453
454 getActionDefinitionsBuilder(Opcode: G_INTTOPTR)
455 .legalForCartesianProduct(Types0: allPtrs, Types1: allIntScalars)
456 .legalIf(
457 Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allPtrs), P1: typeOfExtendedScalars(TypeIdx: 1, IsExtendedInts)))
458 .legalIf(Predicate: [](const LegalityQuery &Query) {
459 const LLT DstTy = Query.Types[0];
460 const LLT SrcTy = Query.Types[1];
461 return DstTy.isPointerVector() && SrcTy.isVector() &&
462 !SrcTy.isPointer() &&
463 DstTy.getNumElements() == SrcTy.getNumElements();
464 });
465 getActionDefinitionsBuilder(Opcode: G_PTRTOINT)
466 .legalForCartesianProduct(Types0: allIntScalars, Types1: allPtrs)
467 .legalIf(
468 Predicate: all(P0: typeOfExtendedScalars(TypeIdx: 0, IsExtendedInts), P1: typeInSet(TypeIdx: 1, TypesInit: allPtrs)))
469 .legalIf(Predicate: [](const LegalityQuery &Query) {
470 const LLT DstTy = Query.Types[0];
471 const LLT SrcTy = Query.Types[1];
472 return SrcTy.isPointerVector() && DstTy.isVector() &&
473 !DstTy.isPointer() &&
474 DstTy.getNumElements() == SrcTy.getNumElements();
475 });
476 getActionDefinitionsBuilder(Opcode: G_PTR_ADD)
477 .legalForCartesianProduct(Types0: allPtrs, Types1: allIntScalars)
478 .legalIf(
479 Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allPtrs), P1: typeOfExtendedScalars(TypeIdx: 1, IsExtendedInts)));
480
481 getActionDefinitionsBuilder(Opcode: G_PTRMASK)
482 .legalForCartesianProduct(Types0: allPtrs, Types1: allIntScalars)
483 .legalIf(
484 Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allPtrs), P1: typeOfExtendedScalars(TypeIdx: 1, IsExtendedInts)))
485 .legalIf(Predicate: [](const LegalityQuery &Query) {
486 const LLT PtrTy = Query.Types[0];
487 const LLT MaskTy = Query.Types[1];
488 return PtrTy.isPointerVector() && MaskTy.isVector() &&
489 !MaskTy.isPointer() &&
490 PtrTy.getNumElements() == MaskTy.getNumElements();
491 });
492
493 // ST.canDirectlyComparePointers() for pointer args is supported in
494 // legalizeCustom().
495 getActionDefinitionsBuilder(Opcode: G_ICMP)
496 .unsupportedIf(Predicate: LegalityPredicates::any(
497 P0: all(P0: typeIs(TypeIdx: 0, TypesInit: p9), P1: typeInSet(TypeIdx: 1, TypesInit: allPtrs), args: typeIsNot(TypeIdx: 1, Type: p9)),
498 P1: all(P0: typeInSet(TypeIdx: 0, TypesInit: allPtrs), P1: typeIsNot(TypeIdx: 0, Type: p9), args: typeIs(TypeIdx: 1, TypesInit: p9))))
499 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 1, Size: MaxVectorSize),
500 Mutation: LegalizeMutations::changeElementCountTo(
501 TypeIdx: 1, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
502 .legalIf(Predicate: [IsExtendedInts](const LegalityQuery &Query) {
503 const LLT Ty = Query.Types[1];
504 return IsExtendedInts && Ty.isValid() && !Ty.isPointerOrPointerVector();
505 })
506 .customIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allBoolScalarsAndVectors),
507 P1: typeInSet(TypeIdx: 1, TypesInit: allPtrsScalarsAndVectors)));
508
509 getActionDefinitionsBuilder(Opcode: G_FCMP)
510 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 1, Size: MaxVectorSize),
511 Mutation: LegalizeMutations::changeElementCountTo(
512 TypeIdx: 1, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
513 .legalIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allBoolScalarsAndVectors),
514 P1: typeInSet(TypeIdx: 1, TypesInit: allFloatScalarsAndVectors)));
515
516 getActionDefinitionsBuilder(Opcodes: {G_ATOMICRMW_OR, G_ATOMICRMW_ADD, G_ATOMICRMW_AND,
517 G_ATOMICRMW_MAX, G_ATOMICRMW_MIN,
518 G_ATOMICRMW_SUB, G_ATOMICRMW_XOR,
519 G_ATOMICRMW_UMAX, G_ATOMICRMW_UMIN})
520 .legalForCartesianProduct(Types0: allIntScalars, Types1: allPtrs);
521
522 getActionDefinitionsBuilder(
523 Opcodes: {G_ATOMICRMW_FADD, G_ATOMICRMW_FSUB, G_ATOMICRMW_FMIN, G_ATOMICRMW_FMAX})
524 .legalForCartesianProduct(Types0: allFloatScalarsAndF16Vector2AndVector4s,
525 Types1: allPtrs);
526
527 getActionDefinitionsBuilder(Opcode: G_ATOMICRMW_XCHG)
528 .legalForCartesianProduct(Types0: allFloatAndIntScalarsAndPtrs, Types1: allPtrs);
529
530 getActionDefinitionsBuilder(Opcode: G_ATOMIC_CMPXCHG_WITH_SUCCESS).lower();
531 // TODO: add proper legalization rules.
532 getActionDefinitionsBuilder(Opcode: G_ATOMIC_CMPXCHG).alwaysLegal();
533 getActionDefinitionsBuilder(Opcode: G_PREFETCH).alwaysLegal();
534
535 getActionDefinitionsBuilder(Opcodes: {G_UADDO, G_USUBO, G_UMULO, G_SMULO})
536 .alwaysLegal();
537
538 getActionDefinitionsBuilder(Opcodes: {G_SADDO, G_SSUBO}).lower();
539
540 // Lowering widens s64 to s128, which needs
541 // SPV_ALTERA_arbitrary_precision_integers. Mark s64 unsupported otherwise.
542 auto &MulFix = getActionDefinitionsBuilder(Opcodes: {G_SMULFIX, G_UMULFIX});
543 if (!HasArbitraryPrecisionInts)
544 MulFix.unsupportedFor(Types: {s64});
545 MulFix.lower();
546
547 getActionDefinitionsBuilder(Opcodes: {G_LROUND, G_LLROUND})
548 .legalForCartesianProduct(Types0: allIntScalarsAndVectors,
549 Types1: allFloatScalarsAndVectors);
550
551 // FP conversions.
552 getActionDefinitionsBuilder(Opcodes: {G_FPTRUNC, G_FPEXT})
553 .legalForCartesianProduct(Types: allFloatScalarsAndVectors);
554
555 // Pointer-handling.
556 getActionDefinitionsBuilder(Opcode: G_FRAME_INDEX).legalFor(Types: {p0});
557
558 getActionDefinitionsBuilder(Opcode: G_GLOBAL_VALUE).legalFor(Types: allPtrs);
559
560 // Control-flow. In some cases (e.g. constants) s1 may be promoted to s32.
561 getActionDefinitionsBuilder(Opcode: G_BR).alwaysLegal();
562 getActionDefinitionsBuilder(Opcode: G_BRCOND).legalFor(Types: {s1, s32});
563
564 getActionDefinitionsBuilder(Opcode: G_FFREXP).legalForCartesianProduct(
565 Types0: allFloatScalarsAndVectors, Types1: {s32, v2s32, v3s32, v4s32, v8s32, v16s32});
566
567 // TODO: Review the target OpenCL and GLSL Extended Instruction Set specs to
568 // tighten these requirements. Many of these math functions are only legal on
569 // specific bitwidths, so they are not selectable for
570 // allFloatScalarsAndVectors.
571 // clang-format off
572 getActionDefinitionsBuilder(Opcodes: {G_STRICT_FSQRT,
573 G_FPOW,
574 G_FEXP,
575 G_FMODF,
576 G_FSINCOS,
577 G_FEXP2,
578 G_FEXP10,
579 G_FLOG,
580 G_FLOG2,
581 G_FLOG10,
582 G_FABS,
583 G_FMINNUM,
584 G_FMAXNUM,
585 G_FCEIL,
586 G_FCOS,
587 G_FSIN,
588 G_FTAN,
589 G_FACOS,
590 G_FASIN,
591 G_FATAN,
592 G_FATAN2,
593 G_FCOSH,
594 G_FSINH,
595 G_FTANH,
596 G_FSQRT,
597 G_FFLOOR,
598 G_FRINT,
599 G_FNEARBYINT,
600 G_INTRINSIC_ROUND,
601 G_INTRINSIC_TRUNC,
602 G_FMINIMUM,
603 G_FMAXIMUM,
604 G_INTRINSIC_ROUNDEVEN})
605 .legalFor(Types: allFloatScalars)
606 .legalFor(Types: allowedFloatVectorTypes)
607 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
608 Mutation: LegalizeMutations::changeElementCountTo(
609 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
610 .moreElementsToNextPow2(TypeIdx: 0);
611 // clang-format on
612
613 getActionDefinitionsBuilder(Opcode: G_FCOPYSIGN)
614 .legalForCartesianProduct(Types0: allFloatScalarsAndVectors,
615 Types1: allFloatScalarsAndVectors);
616
617 getActionDefinitionsBuilder(Opcode: G_FPOWI).legalForCartesianProduct(
618 Types0: allFloatScalarsAndVectors, Types1: allIntScalarsAndVectors);
619
620 if (ST.canUseExtInstSet(E: SPIRV::InstructionSet::OpenCL_std)) {
621 getActionDefinitionsBuilder(
622 Opcodes: {G_CTTZ, G_CTTZ_ZERO_POISON, G_CTLZ, G_CTLZ_ZERO_POISON})
623 .legalForCartesianProduct(Types0: allIntScalarsAndVectors,
624 Types1: allIntScalarsAndVectors);
625
626 // Struct return types become a single scalar, so cannot easily legalize.
627 getActionDefinitionsBuilder(Opcodes: {G_SMULH, G_UMULH}).alwaysLegal();
628 }
629
630 getActionDefinitionsBuilder(Opcode: G_IS_FPCLASS).custom();
631
632 getActionDefinitionsBuilder(Opcodes: {G_INTRINSIC, G_INTRINSIC_CONVERGENT,
633 G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS})
634 .alwaysLegal();
635 getActionDefinitionsBuilder(Opcode: G_FENCE).alwaysLegal();
636 getActionDefinitionsBuilder(Opcodes: {G_TRAP, G_DEBUGTRAP, G_UBSANTRAP}).alwaysLegal();
637
638 verify(MII: *ST.getInstrInfo());
639}
640
641static bool legalizeExtractVectorElt(LegalizerHelper &Helper,
642 MachineInstr &MI) {
643 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
644 Register DstReg = MI.getOperand(i: 0).getReg();
645 Register SrcReg = MI.getOperand(i: 1).getReg();
646 Register IdxReg = MI.getOperand(i: 2).getReg();
647
648 MIRBuilder
649 .buildIntrinsic(ID: Intrinsic::spv_extractelt, Res: ArrayRef<Register>{DstReg})
650 .addUse(RegNo: SrcReg)
651 .addUse(RegNo: IdxReg);
652 MI.eraseFromParent();
653 return true;
654}
655
656static bool legalizeInsertVectorElt(LegalizerHelper &Helper, MachineInstr &MI) {
657 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
658 Register DstReg = MI.getOperand(i: 0).getReg();
659 Register SrcReg = MI.getOperand(i: 1).getReg();
660 Register ValReg = MI.getOperand(i: 2).getReg();
661 Register IdxReg = MI.getOperand(i: 3).getReg();
662
663 MIRBuilder
664 .buildIntrinsic(ID: Intrinsic::spv_insertelt, Res: ArrayRef<Register>{DstReg})
665 .addUse(RegNo: SrcReg)
666 .addUse(RegNo: ValReg)
667 .addUse(RegNo: IdxReg);
668 MI.eraseFromParent();
669 return true;
670}
671
672static Register convertPtrToInt(Register Reg, LLT ConvTy, SPIRVTypeInst SpvType,
673 LegalizerHelper &Helper,
674 MachineRegisterInfo &MRI,
675 SPIRVGlobalRegistry *GR) {
676 Register ConvReg = MRI.createGenericVirtualRegister(Ty: ConvTy);
677 MRI.setRegClass(Reg: ConvReg, RC: GR->getRegClass(SpvType));
678 GR->assignSPIRVTypeToVReg(Type: SpvType, VReg: ConvReg, MF: Helper.MIRBuilder.getMF());
679 Helper.MIRBuilder.buildInstr(Opcode: TargetOpcode::G_PTRTOINT)
680 .addDef(RegNo: ConvReg)
681 .addUse(RegNo: Reg);
682 return ConvReg;
683}
684
685static bool needsVectorLegalization(const LLT &Ty, const SPIRVSubtarget &ST) {
686 if (!Ty.isVector() ||
687 ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_long_vector))
688 return false;
689 unsigned NumElements = Ty.getNumElements();
690 unsigned MaxVectorSize = ST.isShader() ? 4 : 16;
691 return (NumElements > 4 && !isPowerOf2_32(Value: NumElements)) ||
692 NumElements > MaxVectorSize;
693}
694
695static bool legalizeLoad(LegalizerHelper &Helper, MachineInstr &MI,
696 SPIRVGlobalRegistry *GR) {
697 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
698 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
699 Register DstReg = MI.getOperand(i: 0).getReg();
700 Register PtrReg = MI.getOperand(i: 1).getReg();
701 LLT DstTy = MRI.getType(Reg: DstReg);
702
703 if (!DstTy.isVector())
704 return true;
705
706 const SPIRVSubtarget &ST = MI.getMF()->getSubtarget<SPIRVSubtarget>();
707 if (!needsVectorLegalization(Ty: DstTy, ST))
708 return true;
709
710 SmallVector<Register, 8> SplitRegs;
711 LLT EltTy = DstTy.getElementType();
712 unsigned NumElts = DstTy.getNumElements();
713
714 LLT PtrTy = MRI.getType(Reg: PtrReg);
715 auto Zero = MIRBuilder.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: 0);
716
717 for (unsigned i = 0; i < NumElts; ++i) {
718 auto Idx = MIRBuilder.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: i);
719 Register EltPtr = MRI.createGenericVirtualRegister(Ty: PtrTy);
720
721 MIRBuilder.buildIntrinsic(ID: Intrinsic::spv_gep, Res: ArrayRef<Register>{EltPtr})
722 .addImm(Val: 1) // InBounds
723 .addUse(RegNo: PtrReg)
724 .addUse(RegNo: Zero.getReg(Idx: 0))
725 .addUse(RegNo: Idx.getReg(Idx: 0));
726
727 MachinePointerInfo EltPtrInfo;
728 Align EltAlign = Align(1);
729 if (!MI.memoperands_empty()) {
730 MachineMemOperand *MMO = *MI.memoperands_begin();
731 EltPtrInfo =
732 MMO->getPointerInfo().getWithOffset(O: i * EltTy.getSizeInBytes());
733 EltAlign = commonAlignment(A: MMO->getAlign(), Offset: i * EltTy.getSizeInBytes());
734 }
735
736 Register EltReg = MRI.createGenericVirtualRegister(Ty: EltTy);
737 MIRBuilder.buildLoad(Res: EltReg, Addr: EltPtr, PtrInfo: EltPtrInfo, Alignment: EltAlign);
738 SplitRegs.push_back(Elt: EltReg);
739 }
740
741 MIRBuilder.buildBuildVector(Res: DstReg, Ops: SplitRegs);
742 MI.eraseFromParent();
743 return true;
744}
745
746static bool legalizeStore(LegalizerHelper &Helper, MachineInstr &MI,
747 SPIRVGlobalRegistry *GR) {
748 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
749 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
750 Register ValReg = MI.getOperand(i: 0).getReg();
751 Register PtrReg = MI.getOperand(i: 1).getReg();
752 LLT ValTy = MRI.getType(Reg: ValReg);
753
754 assert(ValTy.isVector() && "Expected vector store");
755
756 SmallVector<Register, 8> SplitRegs;
757 LLT EltTy = ValTy.getElementType();
758 unsigned NumElts = ValTy.getNumElements();
759
760 for (unsigned i = 0; i < NumElts; ++i)
761 SplitRegs.push_back(Elt: MRI.createGenericVirtualRegister(Ty: EltTy));
762
763 MIRBuilder.buildUnmerge(Res: SplitRegs, Op: ValReg);
764
765 LLT PtrTy = MRI.getType(Reg: PtrReg);
766 auto Zero = MIRBuilder.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: 0);
767
768 for (unsigned i = 0; i < NumElts; ++i) {
769 auto Idx = MIRBuilder.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: i);
770 Register EltPtr = MRI.createGenericVirtualRegister(Ty: PtrTy);
771
772 MIRBuilder.buildIntrinsic(ID: Intrinsic::spv_gep, Res: ArrayRef<Register>{EltPtr})
773 .addImm(Val: 1) // InBounds
774 .addUse(RegNo: PtrReg)
775 .addUse(RegNo: Zero.getReg(Idx: 0))
776 .addUse(RegNo: Idx.getReg(Idx: 0));
777
778 MachinePointerInfo EltPtrInfo;
779 Align EltAlign = Align(1);
780 if (!MI.memoperands_empty()) {
781 MachineMemOperand *MMO = *MI.memoperands_begin();
782 EltPtrInfo =
783 MMO->getPointerInfo().getWithOffset(O: i * EltTy.getSizeInBytes());
784 EltAlign = commonAlignment(A: MMO->getAlign(), Offset: i * EltTy.getSizeInBytes());
785 }
786
787 MIRBuilder.buildStore(Val: SplitRegs[i], Addr: EltPtr, PtrInfo: EltPtrInfo, Alignment: EltAlign);
788 }
789
790 MI.eraseFromParent();
791 return true;
792}
793
794bool SPIRVLegalizerInfo::legalizeCustom(
795 LegalizerHelper &Helper, MachineInstr &MI,
796 LostDebugLocObserver &LocObserver) const {
797 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
798 switch (MI.getOpcode()) {
799 default:
800 // TODO: implement legalization for other opcodes.
801 return true;
802 case TargetOpcode::G_BITCAST:
803 return legalizeBitcast(Helper, MI);
804 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
805 return legalizeExtractVectorElt(Helper, MI);
806 case TargetOpcode::G_INSERT_VECTOR_ELT:
807 return legalizeInsertVectorElt(Helper, MI);
808 case TargetOpcode::G_INTRINSIC:
809 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
810 return legalizeIntrinsic(Helper, MI);
811 case TargetOpcode::G_IS_FPCLASS:
812 return legalizeIsFPClass(Helper, MI, LocObserver);
813 case TargetOpcode::G_ICMP: {
814 auto &Op0 = MI.getOperand(i: 2);
815 auto &Op1 = MI.getOperand(i: 3);
816 Register Reg0 = Op0.getReg();
817 Register Reg1 = Op1.getReg();
818 CmpInst::Predicate Cond =
819 static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
820 if ((!ST->canDirectlyComparePointers() ||
821 (Cond != CmpInst::ICMP_EQ && Cond != CmpInst::ICMP_NE)) &&
822 MRI.getType(Reg: Reg0).isPointer() && MRI.getType(Reg: Reg1).isPointer()) {
823 LLT ConvT = LLT::scalar(SizeInBits: ST->getPointerSize());
824 Type *LLVMTy = IntegerType::get(C&: MI.getMF()->getFunction().getContext(),
825 NumBits: ST->getPointerSize());
826 SPIRVTypeInst SpirvTy = GR->getOrCreateSPIRVType(
827 Type: LLVMTy, MIRBuilder&: Helper.MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
828 Op0.setReg(convertPtrToInt(Reg: Reg0, ConvTy: ConvT, SpvType: SpirvTy, Helper, MRI, GR));
829 Op1.setReg(convertPtrToInt(Reg: Reg1, ConvTy: ConvT, SpvType: SpirvTy, Helper, MRI, GR));
830 }
831 return true;
832 }
833 case TargetOpcode::G_LOAD:
834 return legalizeLoad(Helper, MI, GR);
835 case TargetOpcode::G_STORE:
836 return legalizeStore(Helper, MI, GR);
837 }
838}
839
840static MachineInstrBuilder
841createStackTemporaryForVector(LegalizerHelper &Helper, SPIRVGlobalRegistry *GR,
842 Register SrcReg, LLT SrcTy,
843 MachinePointerInfo &PtrInfo, Align &VecAlign) {
844 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
845 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
846
847 VecAlign = Helper.getStackTemporaryAlignment(Type: SrcTy);
848 auto StackTemp = Helper.createStackTemporary(
849 Bytes: TypeSize::getFixed(ExactSize: SrcTy.getSizeInBytes()), Alignment: VecAlign, PtrInfo);
850
851 // Set the type of StackTemp to a pointer to an array of the element type.
852 SPIRVTypeInst SpvSrcTy = GR->getSPIRVTypeForVReg(VReg: SrcReg);
853 SPIRVTypeInst EltSpvTy = GR->getScalarOrVectorComponentType(Type: SpvSrcTy);
854 const Type *LLVMEltTy = GR->getTypeForSPIRVType(Ty: EltSpvTy);
855 const Type *LLVMArrTy =
856 ArrayType::get(ElementType: const_cast<Type *>(LLVMEltTy), NumElements: SrcTy.getNumElements());
857 SPIRVTypeInst ArrSpvTy = GR->getOrCreateSPIRVType(
858 Type: LLVMArrTy, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
859 SPIRVTypeInst PtrToArrSpvTy = GR->getOrCreateSPIRVPointerType(
860 BaseType: ArrSpvTy, MIRBuilder, SC: SPIRV::StorageClass::Function);
861
862 Register StackReg = StackTemp.getReg(Idx: 0);
863 MRI.setRegClass(Reg: StackReg, RC: GR->getRegClass(SpvType: PtrToArrSpvTy));
864 GR->assignSPIRVTypeToVReg(Type: PtrToArrSpvTy, VReg: StackReg, MF: MIRBuilder.getMF());
865
866 return StackTemp;
867}
868
869static bool legalizeSpvBitcast(LegalizerHelper &Helper, MachineInstr &MI,
870 SPIRVGlobalRegistry *GR) {
871 LLVM_DEBUG(dbgs() << "Found a bitcast instruction\n");
872 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
873 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
874 const SPIRVSubtarget &ST = MI.getMF()->getSubtarget<SPIRVSubtarget>();
875
876 Register DstReg = MI.getOperand(i: 0).getReg();
877 Register SrcReg = MI.getOperand(i: 2).getReg();
878 LLT DstTy = MRI.getType(Reg: DstReg);
879 LLT SrcTy = MRI.getType(Reg: SrcReg);
880
881 // If an spv_bitcast needs to be legalized, we convert it to G_BITCAST to
882 // allow using the generic legalization rules.
883 if (needsVectorLegalization(Ty: DstTy, ST) ||
884 needsVectorLegalization(Ty: SrcTy, ST)) {
885 LLVM_DEBUG(dbgs() << "Replacing with a G_BITCAST\n");
886 MIRBuilder.buildBitcast(Dst: DstReg, Src: SrcReg);
887 MI.eraseFromParent();
888 }
889 return true;
890}
891
892static bool legalizeSpvInsertElt(LegalizerHelper &Helper, MachineInstr &MI,
893 SPIRVGlobalRegistry *GR) {
894 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
895 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
896 const SPIRVSubtarget &ST = MI.getMF()->getSubtarget<SPIRVSubtarget>();
897
898 Register DstReg = MI.getOperand(i: 0).getReg();
899 LLT DstTy = MRI.getType(Reg: DstReg);
900
901 if (needsVectorLegalization(Ty: DstTy, ST)) {
902 Register SrcReg = MI.getOperand(i: 2).getReg();
903 Register ValReg = MI.getOperand(i: 3).getReg();
904 LLT SrcTy = MRI.getType(Reg: SrcReg);
905 MachineOperand &IdxOperand = MI.getOperand(i: 4);
906
907 if (getImm(MO: IdxOperand, MRI: &MRI)) {
908 uint64_t IdxVal = foldImm(MO: IdxOperand, MRI: &MRI);
909 if (IdxVal < SrcTy.getNumElements()) {
910 SmallVector<Register, 8> Regs;
911 SPIRVTypeInst ElementType =
912 GR->getScalarOrVectorComponentType(Type: GR->getSPIRVTypeForVReg(VReg: DstReg));
913 LLT ElementLLTTy = GR->getRegType(SpvType: ElementType);
914 for (unsigned I = 0, E = SrcTy.getNumElements(); I < E; ++I) {
915 Register Reg = MRI.createGenericVirtualRegister(Ty: ElementLLTTy);
916 MRI.setRegClass(Reg, RC: GR->getRegClass(SpvType: ElementType));
917 GR->assignSPIRVTypeToVReg(Type: ElementType, VReg: Reg, MF: *MI.getMF());
918 Regs.push_back(Elt: Reg);
919 }
920 MIRBuilder.buildUnmerge(Res: Regs, Op: SrcReg);
921 Regs[IdxVal] = ValReg;
922 MIRBuilder.buildBuildVector(Res: DstReg, Ops: Regs);
923 MI.eraseFromParent();
924 return true;
925 }
926 }
927
928 LLT EltTy = SrcTy.getElementType();
929 Align VecAlign;
930 MachinePointerInfo PtrInfo;
931 auto StackTemp = createStackTemporaryForVector(Helper, GR, SrcReg, SrcTy,
932 PtrInfo, VecAlign);
933
934 MIRBuilder.buildStore(Val: SrcReg, Addr: StackTemp, PtrInfo, Alignment: VecAlign);
935
936 Register IdxReg = IdxOperand.getReg();
937 LLT PtrTy = MRI.getType(Reg: StackTemp.getReg(Idx: 0));
938 Register EltPtr = MRI.createGenericVirtualRegister(Ty: PtrTy);
939 auto Zero = MIRBuilder.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: 0);
940
941 MIRBuilder.buildIntrinsic(ID: Intrinsic::spv_gep, Res: ArrayRef<Register>{EltPtr})
942 .addImm(Val: 1) // InBounds
943 .addUse(RegNo: StackTemp.getReg(Idx: 0))
944 .addUse(RegNo: Zero.getReg(Idx: 0))
945 .addUse(RegNo: IdxReg);
946
947 MachinePointerInfo EltPtrInfo = MachinePointerInfo(PtrTy.getAddressSpace());
948 Align EltAlign = Helper.getStackTemporaryAlignment(Type: EltTy);
949 MIRBuilder.buildStore(Val: ValReg, Addr: EltPtr, PtrInfo: EltPtrInfo, Alignment: EltAlign);
950
951 MIRBuilder.buildLoad(Res: DstReg, Addr: StackTemp, PtrInfo, Alignment: VecAlign);
952 MI.eraseFromParent();
953 return true;
954 }
955 return true;
956}
957
958static bool legalizeSpvExtractElt(LegalizerHelper &Helper, MachineInstr &MI,
959 SPIRVGlobalRegistry *GR) {
960 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
961 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
962 const SPIRVSubtarget &ST = MI.getMF()->getSubtarget<SPIRVSubtarget>();
963
964 Register SrcReg = MI.getOperand(i: 2).getReg();
965 LLT SrcTy = MRI.getType(Reg: SrcReg);
966
967 if (needsVectorLegalization(Ty: SrcTy, ST)) {
968 Register DstReg = MI.getOperand(i: 0).getReg();
969 MachineOperand &IdxOperand = MI.getOperand(i: 3);
970
971 if (getImm(MO: IdxOperand, MRI: &MRI)) {
972 uint64_t IdxVal = foldImm(MO: IdxOperand, MRI: &MRI);
973 if (IdxVal < SrcTy.getNumElements()) {
974 LLT DstTy = MRI.getType(Reg: DstReg);
975 SmallVector<Register, 8> Regs;
976 SPIRVTypeInst DstSpvTy = GR->getSPIRVTypeForVReg(VReg: DstReg);
977 for (unsigned I = 0, E = SrcTy.getNumElements(); I < E; ++I) {
978 if (I == IdxVal) {
979 Regs.push_back(Elt: DstReg);
980 } else {
981 Register Reg = MRI.createGenericVirtualRegister(Ty: DstTy);
982 MRI.setRegClass(Reg, RC: GR->getRegClass(SpvType: DstSpvTy));
983 GR->assignSPIRVTypeToVReg(Type: DstSpvTy, VReg: Reg, MF: *MI.getMF());
984 Regs.push_back(Elt: Reg);
985 }
986 }
987 MIRBuilder.buildUnmerge(Res: Regs, Op: SrcReg);
988 MI.eraseFromParent();
989 return true;
990 }
991 }
992
993 LLT EltTy = SrcTy.getElementType();
994 Align VecAlign;
995 MachinePointerInfo PtrInfo;
996 auto StackTemp = createStackTemporaryForVector(Helper, GR, SrcReg, SrcTy,
997 PtrInfo, VecAlign);
998
999 MIRBuilder.buildStore(Val: SrcReg, Addr: StackTemp, PtrInfo, Alignment: VecAlign);
1000
1001 Register IdxReg = IdxOperand.getReg();
1002 LLT PtrTy = MRI.getType(Reg: StackTemp.getReg(Idx: 0));
1003 Register EltPtr = MRI.createGenericVirtualRegister(Ty: PtrTy);
1004 auto Zero = MIRBuilder.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: 0);
1005
1006 MIRBuilder.buildIntrinsic(ID: Intrinsic::spv_gep, Res: ArrayRef<Register>{EltPtr})
1007 .addImm(Val: 1) // InBounds
1008 .addUse(RegNo: StackTemp.getReg(Idx: 0))
1009 .addUse(RegNo: Zero.getReg(Idx: 0))
1010 .addUse(RegNo: IdxReg);
1011
1012 MachinePointerInfo EltPtrInfo = MachinePointerInfo(PtrTy.getAddressSpace());
1013 Align EltAlign = Helper.getStackTemporaryAlignment(Type: EltTy);
1014 MIRBuilder.buildLoad(Res: DstReg, Addr: EltPtr, PtrInfo: EltPtrInfo, Alignment: EltAlign);
1015
1016 MI.eraseFromParent();
1017 return true;
1018 }
1019 return true;
1020}
1021
1022static bool legalizeSpvConstComposite(LegalizerHelper &Helper, MachineInstr &MI,
1023 SPIRVGlobalRegistry *GR) {
1024 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1025 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
1026 const SPIRVSubtarget &ST = MI.getMF()->getSubtarget<SPIRVSubtarget>();
1027
1028 Register DstReg = MI.getOperand(i: 0).getReg();
1029 LLT DstTy = MRI.getType(Reg: DstReg);
1030
1031 if (!needsVectorLegalization(Ty: DstTy, ST))
1032 return true;
1033
1034 SmallVector<Register, 8> SrcRegs;
1035 if (MI.getNumOperands() == 2) {
1036 // The "null" case: no values are attached.
1037 LLT EltTy = DstTy.getElementType();
1038 auto Zero = MIRBuilder.buildConstant(Res: EltTy, Val: 0);
1039 SPIRVTypeInst SpvDstTy = GR->getSPIRVTypeForVReg(VReg: DstReg);
1040 SPIRVTypeInst SpvEltTy = GR->getScalarOrVectorComponentType(Type: SpvDstTy);
1041 GR->assignSPIRVTypeToVReg(Type: SpvEltTy, VReg: Zero.getReg(Idx: 0), MF: MIRBuilder.getMF());
1042 for (unsigned i = 0; i < DstTy.getNumElements(); ++i)
1043 SrcRegs.push_back(Elt: Zero.getReg(Idx: 0));
1044 } else {
1045 for (unsigned i = 2; i < MI.getNumOperands(); ++i) {
1046 SrcRegs.push_back(Elt: MI.getOperand(i).getReg());
1047 }
1048 }
1049 MIRBuilder.buildBuildVector(Res: DstReg, Ops: SrcRegs);
1050 MI.eraseFromParent();
1051 return true;
1052}
1053
1054bool SPIRVLegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
1055 MachineInstr &MI) const {
1056 LLVM_DEBUG(dbgs() << "legalizeIntrinsic: " << MI);
1057 auto IntrinsicID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
1058 switch (IntrinsicID) {
1059 case Intrinsic::spv_bitcast:
1060 return legalizeSpvBitcast(Helper, MI, GR);
1061 case Intrinsic::spv_insertelt:
1062 return legalizeSpvInsertElt(Helper, MI, GR);
1063 case Intrinsic::spv_extractelt:
1064 return legalizeSpvExtractElt(Helper, MI, GR);
1065 case Intrinsic::spv_const_composite:
1066 return legalizeSpvConstComposite(Helper, MI, GR);
1067 }
1068 return true;
1069}
1070
1071bool SPIRVLegalizerInfo::legalizeBitcast(LegalizerHelper &Helper,
1072 MachineInstr &MI) const {
1073 // Once the G_BITCAST is using vectors that are allowed, we turn it back into
1074 // an spv_bitcast to avoid verifier problems when the register types are the
1075 // same for the source and the result. Note that the SPIR-V types associated
1076 // with the bitcast can be different even if the register types are the same.
1077 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1078 Register DstReg = MI.getOperand(i: 0).getReg();
1079 Register SrcReg = MI.getOperand(i: 1).getReg();
1080 SmallVector<Register, 1> DstRegs = {DstReg};
1081 MIRBuilder.buildIntrinsic(ID: Intrinsic::spv_bitcast, Res: DstRegs).addUse(RegNo: SrcReg);
1082 MI.eraseFromParent();
1083 return true;
1084}
1085
1086// Note this code was copied from LegalizerHelper::lowerISFPCLASS and adjusted
1087// to ensure that all instructions created during the lowering have SPIR-V types
1088// assigned to them.
1089bool SPIRVLegalizerInfo::legalizeIsFPClass(
1090 LegalizerHelper &Helper, MachineInstr &MI,
1091 LostDebugLocObserver &LocObserver) const {
1092 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
1093 FPClassTest Mask = static_cast<FPClassTest>(MI.getOperand(i: 2).getImm());
1094
1095 auto &MIRBuilder = Helper.MIRBuilder;
1096 auto &MF = MIRBuilder.getMF();
1097 MachineRegisterInfo &MRI = MF.getRegInfo();
1098
1099 Type *LLVMDstTy =
1100 IntegerType::get(C&: MIRBuilder.getContext(), NumBits: DstTy.getScalarSizeInBits());
1101 if (DstTy.isVector())
1102 LLVMDstTy = VectorType::get(ElementType: LLVMDstTy, EC: DstTy.getElementCount());
1103 SPIRVTypeInst SPIRVDstTy = GR->getOrCreateSPIRVType(
1104 Type: LLVMDstTy, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite,
1105 /*EmitIR*/ true);
1106
1107 unsigned BitSize = SrcTy.getScalarSizeInBits();
1108 const fltSemantics &Semantics = getFltSemanticForLLT(Ty: SrcTy.getScalarType());
1109
1110 LLT IntTy = LLT::scalar(SizeInBits: BitSize);
1111 Type *LLVMIntTy = IntegerType::get(C&: MIRBuilder.getContext(), NumBits: BitSize);
1112 if (SrcTy.isVector()) {
1113 IntTy = LLT::vector(EC: SrcTy.getElementCount(), ScalarTy: IntTy);
1114 LLVMIntTy = VectorType::get(ElementType: LLVMIntTy, EC: SrcTy.getElementCount());
1115 }
1116 SPIRVTypeInst SPIRVIntTy = GR->getOrCreateSPIRVType(
1117 Type: LLVMIntTy, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite,
1118 /*EmitIR*/ true);
1119
1120 // Clang doesn't support capture of structured bindings:
1121 LLT DstTyCopy = DstTy;
1122 const auto assignSPIRVTy = [&](MachineInstrBuilder &&MI) {
1123 // Assign this MI's (assumed only) destination to one of the two types we
1124 // expect: either the G_IS_FPCLASS's destination type, or the integer type
1125 // bitcast from the source type.
1126 LLT MITy = MRI.getType(Reg: MI.getReg(Idx: 0));
1127 assert((MITy == IntTy || MITy == DstTyCopy) &&
1128 "Unexpected LLT type while lowering G_IS_FPCLASS");
1129 SPIRVTypeInst SPVTy = MITy == IntTy ? SPIRVIntTy : SPIRVDstTy;
1130 GR->assignSPIRVTypeToVReg(Type: SPVTy, VReg: MI.getReg(Idx: 0), MF);
1131 return MI;
1132 };
1133
1134 // Helper to build and assign a constant in one go
1135 const auto buildSPIRVConstant = [&](LLT Ty, auto &&C) -> MachineInstrBuilder {
1136 if (!Ty.isFixedVector())
1137 return assignSPIRVTy(MIRBuilder.buildConstant(Ty, C));
1138 auto ScalarC = MIRBuilder.buildConstant(Ty.getScalarType(), C);
1139 assert((Ty == IntTy || Ty == DstTyCopy) &&
1140 "Unexpected LLT type while lowering constant for G_IS_FPCLASS");
1141 SPIRVTypeInst VecEltTy = GR->getOrCreateSPIRVType(
1142 Type: (Ty == IntTy ? LLVMIntTy : LLVMDstTy)->getScalarType(), MIRBuilder,
1143 AQ: SPIRV::AccessQualifier::ReadWrite,
1144 /*EmitIR*/ true);
1145 GR->assignSPIRVTypeToVReg(Type: VecEltTy, VReg: ScalarC.getReg(0), MF);
1146 return assignSPIRVTy(MIRBuilder.buildSplatBuildVector(Res: Ty, Src: ScalarC));
1147 };
1148
1149 if (Mask == fcNone) {
1150 MIRBuilder.buildCopy(Res: DstReg, Op: buildSPIRVConstant(DstTy, 0));
1151 MI.eraseFromParent();
1152 return true;
1153 }
1154 if (Mask == fcAllFlags) {
1155 MIRBuilder.buildCopy(Res: DstReg, Op: buildSPIRVConstant(DstTy, 1));
1156 MI.eraseFromParent();
1157 return true;
1158 }
1159
1160 // Note that rather than creating a COPY here (between a floating-point and
1161 // integer type of the same size) we create a SPIR-V bitcast immediately. We
1162 // can't create a G_BITCAST because the LLTs are the same, and we can't seem
1163 // to correctly lower COPYs to SPIR-V bitcasts at this moment.
1164 Register ResVReg = MRI.createGenericVirtualRegister(Ty: IntTy);
1165 MRI.setRegClass(Reg: ResVReg, RC: GR->getRegClass(SpvType: SPIRVIntTy));
1166 GR->assignSPIRVTypeToVReg(Type: SPIRVIntTy, VReg: ResVReg, MF: Helper.MIRBuilder.getMF());
1167 auto AsInt = MIRBuilder.buildInstr(Opcode: SPIRV::OpBitcast)
1168 .addDef(RegNo: ResVReg)
1169 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: SPIRVIntTy))
1170 .addUse(RegNo: SrcReg);
1171 AsInt = assignSPIRVTy(std::move(AsInt));
1172
1173 // Various masks.
1174 APInt SignBit = APInt::getSignMask(BitWidth: BitSize);
1175 APInt ValueMask = APInt::getSignedMaxValue(numBits: BitSize); // All bits but sign.
1176 APInt Inf = APFloat::getInf(Sem: Semantics).bitcastToAPInt(); // Exp and int bit.
1177 APInt ExpMask = Inf;
1178 APInt AllOneMantissa = APFloat::getLargest(Sem: Semantics).bitcastToAPInt() & ~Inf;
1179 APInt QNaNBitMask =
1180 APInt::getOneBitSet(numBits: BitSize, BitNo: AllOneMantissa.getActiveBits() - 1);
1181 APInt InversionMask = APInt::getAllOnes(numBits: DstTy.getScalarSizeInBits());
1182
1183 auto SignBitC = buildSPIRVConstant(IntTy, SignBit);
1184 auto ValueMaskC = buildSPIRVConstant(IntTy, ValueMask);
1185 auto InfC = buildSPIRVConstant(IntTy, Inf);
1186 auto ExpMaskC = buildSPIRVConstant(IntTy, ExpMask);
1187 auto ZeroC = buildSPIRVConstant(IntTy, 0);
1188
1189 auto Abs = assignSPIRVTy(MIRBuilder.buildAnd(Dst: IntTy, Src0: AsInt, Src1: ValueMaskC));
1190 auto Sign = assignSPIRVTy(
1191 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_NE, Res: DstTy, Op0: AsInt, Op1: Abs));
1192
1193 auto Res = buildSPIRVConstant(DstTy, 0);
1194
1195 const auto appendToRes = [&](MachineInstrBuilder &&ToAppend) {
1196 Res = assignSPIRVTy(
1197 MIRBuilder.buildOr(Dst: DstTyCopy, Src0: Res, Src1: assignSPIRVTy(std::move(ToAppend))));
1198 };
1199
1200 // Tests that involve more than one class should be processed first.
1201 if ((Mask & fcFinite) == fcFinite) {
1202 // finite(V) ==> abs(V) u< exp_mask
1203 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: Abs,
1204 Op1: ExpMaskC));
1205 Mask &= ~fcFinite;
1206 } else if ((Mask & fcFinite) == fcPosFinite) {
1207 // finite(V) && V > 0 ==> V u< exp_mask
1208 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: AsInt,
1209 Op1: ExpMaskC));
1210 Mask &= ~fcPosFinite;
1211 } else if ((Mask & fcFinite) == fcNegFinite) {
1212 // finite(V) && V < 0 ==> abs(V) u< exp_mask && signbit == 1
1213 auto Cmp = assignSPIRVTy(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT,
1214 Res: DstTy, Op0: Abs, Op1: ExpMaskC));
1215 appendToRes(MIRBuilder.buildAnd(Dst: DstTy, Src0: Cmp, Src1: Sign));
1216 Mask &= ~fcNegFinite;
1217 }
1218
1219 if (FPClassTest PartialCheck = Mask & (fcZero | fcSubnormal)) {
1220 // fcZero | fcSubnormal => test all exponent bits are 0
1221 // TODO: Handle sign bit specific cases
1222 // TODO: Handle inverted case
1223 if (PartialCheck == (fcZero | fcSubnormal)) {
1224 auto ExpBits = assignSPIRVTy(MIRBuilder.buildAnd(Dst: IntTy, Src0: AsInt, Src1: ExpMaskC));
1225 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
1226 Op0: ExpBits, Op1: ZeroC));
1227 Mask &= ~PartialCheck;
1228 }
1229 }
1230
1231 // Check for individual classes.
1232 if (FPClassTest PartialCheck = Mask & fcZero) {
1233 if (PartialCheck == fcPosZero)
1234 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
1235 Op0: AsInt, Op1: ZeroC));
1236 else if (PartialCheck == fcZero)
1237 appendToRes(
1238 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy, Op0: Abs, Op1: ZeroC));
1239 else // fcNegZero
1240 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
1241 Op0: AsInt, Op1: SignBitC));
1242 }
1243
1244 if (FPClassTest PartialCheck = Mask & fcSubnormal) {
1245 // issubnormal(V) ==> unsigned(abs(V) - 1) u< (all mantissa bits set)
1246 // issubnormal(V) && V>0 ==> unsigned(V - 1) u< (all mantissa bits set)
1247 auto V = (PartialCheck == fcPosSubnormal) ? AsInt : Abs;
1248 auto OneC = buildSPIRVConstant(IntTy, 1);
1249 auto VMinusOne = MIRBuilder.buildSub(Dst: IntTy, Src0: V, Src1: OneC);
1250 auto SubnormalRes = assignSPIRVTy(
1251 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: VMinusOne,
1252 Op1: buildSPIRVConstant(IntTy, AllOneMantissa)));
1253 if (PartialCheck == fcNegSubnormal)
1254 SubnormalRes = MIRBuilder.buildAnd(Dst: DstTy, Src0: SubnormalRes, Src1: Sign);
1255 appendToRes(std::move(SubnormalRes));
1256 }
1257
1258 if (FPClassTest PartialCheck = Mask & fcInf) {
1259 if (PartialCheck == fcPosInf)
1260 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
1261 Op0: AsInt, Op1: InfC));
1262 else if (PartialCheck == fcInf)
1263 appendToRes(
1264 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy, Op0: Abs, Op1: InfC));
1265 else { // fcNegInf
1266 APInt NegInf = APFloat::getInf(Sem: Semantics, Negative: true).bitcastToAPInt();
1267 auto NegInfC = buildSPIRVConstant(IntTy, NegInf);
1268 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
1269 Op0: AsInt, Op1: NegInfC));
1270 }
1271 }
1272
1273 if (FPClassTest PartialCheck = Mask & fcNan) {
1274 auto InfWithQnanBitC =
1275 buildSPIRVConstant(IntTy, std::move(Inf) | QNaNBitMask);
1276 if (PartialCheck == fcNan) {
1277 // isnan(V) ==> abs(V) u> int(inf)
1278 appendToRes(
1279 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_UGT, Res: DstTy, Op0: Abs, Op1: InfC));
1280 } else if (PartialCheck == fcQNan) {
1281 // isquiet(V) ==> abs(V) u>= (unsigned(Inf) | quiet_bit)
1282 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_UGE, Res: DstTy, Op0: Abs,
1283 Op1: InfWithQnanBitC));
1284 } else { // fcSNan
1285 // issignaling(V) ==> abs(V) u> unsigned(Inf) &&
1286 // abs(V) u< (unsigned(Inf) | quiet_bit)
1287 auto IsNan = assignSPIRVTy(
1288 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_UGT, Res: DstTy, Op0: Abs, Op1: InfC));
1289 auto IsNotQnan = assignSPIRVTy(MIRBuilder.buildICmp(
1290 Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: Abs, Op1: InfWithQnanBitC));
1291 appendToRes(MIRBuilder.buildAnd(Dst: DstTy, Src0: IsNan, Src1: IsNotQnan));
1292 }
1293 }
1294
1295 if (FPClassTest PartialCheck = Mask & fcNormal) {
1296 // isnormal(V) ==> (0 u< exp u< max_exp) ==> (unsigned(exp-1) u<
1297 // (max_exp-1))
1298 APInt ExpLSB = ExpMask & ~(ExpMask.shl(shiftAmt: 1));
1299 auto ExpMinusOne = assignSPIRVTy(
1300 MIRBuilder.buildSub(Dst: IntTy, Src0: Abs, Src1: buildSPIRVConstant(IntTy, ExpLSB)));
1301 APInt MaxExpMinusOne = std::move(ExpMask) - ExpLSB;
1302 auto NormalRes = assignSPIRVTy(
1303 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: ExpMinusOne,
1304 Op1: buildSPIRVConstant(IntTy, MaxExpMinusOne)));
1305 if (PartialCheck == fcNegNormal)
1306 NormalRes = MIRBuilder.buildAnd(Dst: DstTy, Src0: NormalRes, Src1: Sign);
1307 else if (PartialCheck == fcPosNormal) {
1308 auto PosSign = assignSPIRVTy(MIRBuilder.buildXor(
1309 Dst: DstTy, Src0: Sign, Src1: buildSPIRVConstant(DstTy, InversionMask)));
1310 NormalRes = MIRBuilder.buildAnd(Dst: DstTy, Src0: NormalRes, Src1: PosSign);
1311 }
1312 appendToRes(std::move(NormalRes));
1313 }
1314
1315 MIRBuilder.buildCopy(Res: DstReg, Op: Res);
1316 MI.eraseFromParent();
1317 return true;
1318}
1319