1//===- SPIRVLegalizerInfo.cpp --- SPIR-V Legalization Rules ------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the targeting of the Machinelegalizer class for SPIR-V.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SPIRVLegalizerInfo.h"
14#include "SPIRV.h"
15#include "SPIRVGlobalRegistry.h"
16#include "SPIRVSubtarget.h"
17#include "SPIRVUtils.h"
18#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
19#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
20#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
21#include "llvm/CodeGen/MachineInstr.h"
22#include "llvm/CodeGen/MachineRegisterInfo.h"
23#include "llvm/CodeGen/TargetOpcodes.h"
24#include "llvm/IR/IntrinsicsSPIRV.h"
25#include "llvm/Support/Debug.h"
26#include "llvm/Support/MathExtras.h"
27
28using namespace llvm;
29using namespace llvm::LegalizeActions;
30using namespace llvm::LegalityPredicates;
31
32#define DEBUG_TYPE "spirv-legalizer"
33
34LegalityPredicate typeOfExtendedScalars(unsigned TypeIdx, bool IsExtendedInts) {
35 return [IsExtendedInts, TypeIdx](const LegalityQuery &Query) {
36 const LLT Ty = Query.Types[TypeIdx];
37 return IsExtendedInts && Ty.isValid() && Ty.isScalar();
38 };
39}
40
41SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) {
42 using namespace TargetOpcode;
43
44 this->ST = &ST;
45 GR = ST.getSPIRVGlobalRegistry();
46
47 const LLT s1 = LLT::scalar(SizeInBits: 1);
48 const LLT s8 = LLT::scalar(SizeInBits: 8);
49 const LLT s16 = LLT::scalar(SizeInBits: 16);
50 const LLT s32 = LLT::scalar(SizeInBits: 32);
51 const LLT s64 = LLT::scalar(SizeInBits: 64);
52 const LLT s128 = LLT::scalar(SizeInBits: 128);
53
54 const LLT v16s64 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 64);
55 const LLT v16s32 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 32);
56 const LLT v16s16 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 16);
57 const LLT v16s8 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 8);
58 const LLT v16s1 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 1);
59
60 const LLT v8s64 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 64);
61 const LLT v8s32 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 32);
62 const LLT v8s16 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16);
63 const LLT v8s8 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 8);
64 const LLT v8s1 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 1);
65
66 const LLT v4s64 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 64);
67 const LLT v4s32 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32);
68 const LLT v4s16 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16);
69 const LLT v4s8 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 8);
70 const LLT v4s1 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 1);
71
72 const LLT v3s64 = LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 64);
73 const LLT v3s32 = LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 32);
74 const LLT v3s16 = LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 16);
75 const LLT v3s8 = LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 8);
76 const LLT v3s1 = LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 1);
77
78 const LLT v2s64 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64);
79 const LLT v2s32 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32);
80 const LLT v2s16 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16);
81 const LLT v2s8 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 8);
82 const LLT v2s1 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 1);
83
84 const unsigned PSize = ST.getPointerSize();
85 const LLT p0 = LLT::pointer(AddressSpace: 0, SizeInBits: PSize); // Function
86 const LLT p1 = LLT::pointer(AddressSpace: 1, SizeInBits: PSize); // CrossWorkgroup
87 const LLT p2 = LLT::pointer(AddressSpace: 2, SizeInBits: PSize); // UniformConstant
88 const LLT p3 = LLT::pointer(AddressSpace: 3, SizeInBits: PSize); // Workgroup
89 const LLT p4 = LLT::pointer(AddressSpace: 4, SizeInBits: PSize); // Generic
90 const LLT p5 =
91 LLT::pointer(AddressSpace: 5, SizeInBits: PSize); // Input, SPV_INTEL_usm_storage_classes (Device)
92 const LLT p6 = LLT::pointer(AddressSpace: 6, SizeInBits: PSize); // SPV_INTEL_usm_storage_classes (Host)
93 const LLT p7 = LLT::pointer(AddressSpace: 7, SizeInBits: PSize); // Input
94 const LLT p8 = LLT::pointer(AddressSpace: 8, SizeInBits: PSize); // Output
95 const LLT p9 =
96 LLT::pointer(AddressSpace: 9, SizeInBits: PSize); // CodeSectionINTEL, SPV_INTEL_function_pointers
97 const LLT p10 = LLT::pointer(AddressSpace: 10, SizeInBits: PSize); // Private
98 const LLT p11 = LLT::pointer(AddressSpace: 11, SizeInBits: PSize); // StorageBuffer
99 const LLT p12 = LLT::pointer(AddressSpace: 12, SizeInBits: PSize); // Uniform
100 const LLT p13 = LLT::pointer(AddressSpace: 13, SizeInBits: PSize); // PushConstant
101
102 // TODO: remove copy-pasting here by using concatenation in some way.
103 auto allPtrsScalarsAndVectors = {
104 p0, p1, p2, p3, p4, p5, p6, p7, p8,
105 p9, p10, p11, p12, p13, s1, s8, s16, s32,
106 s64, s128, v2s1, v2s8, v2s16, v2s32, v2s64, v3s1, v3s8,
107 v3s16, v3s32, v3s64, v4s1, v4s8, v4s16, v4s32, v4s64, v8s1,
108 v8s8, v8s16, v8s32, v8s64, v16s1, v16s8, v16s16, v16s32, v16s64};
109
110 auto allVectors = {v2s1, v2s8, v2s16, v2s32, v2s64, v3s1, v3s8,
111 v3s16, v3s32, v3s64, v4s1, v4s8, v4s16, v4s32,
112 v4s64, v8s1, v8s8, v8s16, v8s32, v8s64, v16s1,
113 v16s8, v16s16, v16s32, v16s64};
114
115 auto allShaderVectors = {v2s1, v2s8, v2s16, v2s32, v2s64,
116 v3s1, v3s8, v3s16, v3s32, v3s64,
117 v4s1, v4s8, v4s16, v4s32, v4s64};
118
119 auto allScalars = {s1, s8, s16, s32, s64};
120
121 auto allScalarsAndVectors = {
122 s1, s8, s16, s32, s64, s128, v2s1, v2s8,
123 v2s16, v2s32, v2s64, v3s1, v3s8, v3s16, v3s32, v3s64,
124 v4s1, v4s8, v4s16, v4s32, v4s64, v8s1, v8s8, v8s16,
125 v8s32, v8s64, v16s1, v16s8, v16s16, v16s32, v16s64};
126
127 auto allIntScalarsAndVectors = {
128 s8, s16, s32, s64, s128, v2s8, v2s16, v2s32, v2s64,
129 v3s8, v3s16, v3s32, v3s64, v4s8, v4s16, v4s32, v4s64, v8s8,
130 v8s16, v8s32, v8s64, v16s8, v16s16, v16s32, v16s64};
131
132 auto allBoolScalarsAndVectors = {s1, v2s1, v3s1, v4s1, v8s1, v16s1};
133
134 auto allIntScalars = {s8, s16, s32, s64, s128};
135
136 auto allFloatScalarsAndF16Vector2AndVector4s = {s16, s32, s64, v2s16, v4s16};
137
138 auto allFloatScalars = {s16, s32, s64};
139
140 auto allFloatScalarsAndVectors = {
141 s16, s32, s64, v2s16, v2s32, v2s64, v3s16, v3s32, v3s64,
142 v4s16, v4s32, v4s64, v8s16, v8s32, v8s64, v16s16, v16s32, v16s64};
143
144 auto allShaderFloatVectors = {v2s16, v2s32, v2s64, v3s16, v3s32,
145 v3s64, v4s16, v4s32, v4s64};
146
147 auto allFloatVectors = {v2s16, v2s32, v2s64, v3s16, v3s32,
148 v3s64, v4s16, v4s32, v4s64, v8s16,
149 v8s32, v8s64, v16s16, v16s32, v16s64};
150
151 auto &allowedFloatVectorTypes =
152 ST.isShader() ? allShaderFloatVectors : allFloatVectors;
153
154 auto allFloatAndIntScalarsAndPtrs = {s8, s16, s32, s64, p0, p1,
155 p2, p3, p4, p5, p6, p7,
156 p8, p9, p10, p11, p12, p13};
157
158 auto allPtrs = {p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13};
159
160 auto &allowedVectorTypes = ST.isShader() ? allShaderVectors : allVectors;
161
162 bool HasArbitraryPrecisionInts = ST.canUseExtension(
163 E: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers);
164 bool IsExtendedInts =
165 HasArbitraryPrecisionInts ||
166 ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_bit_instructions) ||
167 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_int4);
168 auto extendedScalarsAndVectors =
169 [IsExtendedInts](const LegalityQuery &Query) {
170 const LLT Ty = Query.Types[0];
171 return IsExtendedInts && Ty.isValid() && !Ty.isPointerOrPointerVector();
172 };
173 auto extendedScalarsAndVectorsProduct = [IsExtendedInts](
174 const LegalityQuery &Query) {
175 const LLT Ty1 = Query.Types[0], Ty2 = Query.Types[1];
176 return IsExtendedInts && Ty1.isValid() && Ty2.isValid() &&
177 !Ty1.isPointerOrPointerVector() && !Ty2.isPointerOrPointerVector();
178 };
179 auto extendedPtrsScalarsAndVectors =
180 [IsExtendedInts](const LegalityQuery &Query) {
181 const LLT Ty = Query.Types[0];
182 return IsExtendedInts && Ty.isValid();
183 };
184
185 // The universal validation rules in the SPIR-V specification state that
186 // vector sizes are typically limited to 2, 3, or 4. However, larger vector
187 // sizes (8 and 16) are enabled when the Kernel capability is present. For
188 // shader execution models, vector sizes are strictly limited to 4. In
189 // non-shader contexts, vector sizes of 8 and 16 are also permitted, but
190 // arbitrary sizes (e.g., 6 or 11) are not.
191 uint32_t MaxVectorSize = ST.isShader() ? 4 : 16;
192 LLVM_DEBUG(dbgs() << "MaxVectorSize: " << MaxVectorSize << "\n");
193
194 for (auto Opc : getTypeFoldingSupportedOpcodes()) {
195 switch (Opc) {
196 case G_EXTRACT_VECTOR_ELT:
197 case G_UREM:
198 case G_SREM:
199 case G_UDIV:
200 case G_SDIV:
201 case G_FREM:
202 break;
203 default:
204 getActionDefinitionsBuilder(Opcode: Opc)
205 .customFor(Types: allScalars)
206 .customFor(Types: allowedVectorTypes)
207 .moreElementsToNextPow2(TypeIdx: 0)
208 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
209 Mutation: LegalizeMutations::changeElementCountTo(
210 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
211 .custom();
212 break;
213 }
214 }
215
216 getActionDefinitionsBuilder(Opcodes: {G_UREM, G_SREM, G_SDIV, G_UDIV, G_FREM})
217 .customFor(Types: allScalars)
218 .customFor(Types: allowedVectorTypes)
219 .scalarizeIf(Predicate: numElementsNotPow2(TypeIdx: 0), TypeIdx: 0)
220 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
221 Mutation: LegalizeMutations::changeElementCountTo(
222 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
223 .custom();
224
225 getActionDefinitionsBuilder(Opcodes: {G_FMA, G_STRICT_FMA})
226 .legalFor(Types: allScalars)
227 .legalFor(Types: allowedVectorTypes)
228 .moreElementsToNextPow2(TypeIdx: 0)
229 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
230 Mutation: LegalizeMutations::changeElementCountTo(
231 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
232 .alwaysLegal();
233
234 getActionDefinitionsBuilder(Opcode: G_INTRINSIC_W_SIDE_EFFECTS).custom();
235
236 getActionDefinitionsBuilder(Opcode: G_SHUFFLE_VECTOR)
237 .legalForCartesianProduct(Types0: allowedVectorTypes, Types1: allowedVectorTypes)
238 .moreElementsToNextPow2(TypeIdx: 0)
239 .lowerIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize))
240 .moreElementsToNextPow2(TypeIdx: 1)
241 .lowerIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 1, Size: MaxVectorSize));
242
243 getActionDefinitionsBuilder(Opcode: G_EXTRACT_VECTOR_ELT)
244 .moreElementsToNextPow2(TypeIdx: 1)
245 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 1, Size: MaxVectorSize),
246 Mutation: LegalizeMutations::changeElementCountTo(
247 TypeIdx: 1, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
248 .custom();
249
250 getActionDefinitionsBuilder(Opcode: G_INSERT_VECTOR_ELT)
251 .moreElementsToNextPow2(TypeIdx: 0)
252 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
253 Mutation: LegalizeMutations::changeElementCountTo(
254 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
255 .custom();
256
257 // Illegal G_UNMERGE_VALUES instructions should be handled
258 // during the combine phase.
259 getActionDefinitionsBuilder(Opcode: G_BUILD_VECTOR)
260 .legalIf(Predicate: vectorElementCountIsLessThanOrEqualTo(TypeIdx: 0, Size: MaxVectorSize));
261
262 // When entering the legalizer, there should be no G_BITCAST instructions.
263 // They should all be calls to the `spv_bitcast` intrinsic. The call to
264 // the intrinsic will be converted to a G_BITCAST during legalization if
265 // the vectors are not legal. After using the rules to legalize a G_BITCAST,
266 // we turn it back into a call to the intrinsic with a custom rule to avoid
267 // potential machine verifier failures.
268 getActionDefinitionsBuilder(Opcode: G_BITCAST)
269 .moreElementsToNextPow2(TypeIdx: 0)
270 .moreElementsToNextPow2(TypeIdx: 1)
271 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
272 Mutation: LegalizeMutations::changeElementCountTo(
273 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
274 .lowerIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 1, Size: MaxVectorSize))
275 .custom();
276
277 // If the result is still illegal, the combiner should be able to remove it.
278 getActionDefinitionsBuilder(Opcode: G_CONCAT_VECTORS)
279 .legalForCartesianProduct(Types0: allowedVectorTypes, Types1: allowedVectorTypes);
280
281 getActionDefinitionsBuilder(Opcode: G_SPLAT_VECTOR)
282 .legalFor(Types: allowedVectorTypes)
283 .moreElementsToNextPow2(TypeIdx: 0)
284 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
285 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx: 0, FromTypeIdx: MaxVectorSize))
286 .alwaysLegal();
287
288 // Vector Reduction Operations
289 getActionDefinitionsBuilder(
290 Opcodes: {G_VECREDUCE_SMIN, G_VECREDUCE_SMAX, G_VECREDUCE_UMIN, G_VECREDUCE_UMAX,
291 G_VECREDUCE_ADD, G_VECREDUCE_MUL, G_VECREDUCE_FMUL, G_VECREDUCE_FMIN,
292 G_VECREDUCE_FMAX, G_VECREDUCE_FMINIMUM, G_VECREDUCE_FMAXIMUM,
293 G_VECREDUCE_OR, G_VECREDUCE_AND, G_VECREDUCE_XOR})
294 .legalFor(Types: allowedVectorTypes)
295 .scalarize(TypeIdx: 1)
296 .lower();
297
298 getActionDefinitionsBuilder(Opcodes: {G_VECREDUCE_SEQ_FADD, G_VECREDUCE_SEQ_FMUL})
299 .scalarize(TypeIdx: 2)
300 .lower();
301
302 // Illegal G_UNMERGE_VALUES instructions should be handled
303 // during the combine phase.
304 getActionDefinitionsBuilder(Opcode: G_UNMERGE_VALUES)
305 .legalIf(Predicate: vectorElementCountIsLessThanOrEqualTo(TypeIdx: 1, Size: MaxVectorSize));
306
307 getActionDefinitionsBuilder(Opcodes: {G_MEMCPY, G_MEMCPY_INLINE, G_MEMMOVE})
308 .unsupportedIf(Predicate: LegalityPredicates::any(P0: typeIs(TypeIdx: 0, TypesInit: p9), P1: typeIs(TypeIdx: 1, TypesInit: p9)))
309 .legalIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allPtrs), P1: typeInSet(TypeIdx: 1, TypesInit: allPtrs)));
310
311 getActionDefinitionsBuilder(Opcodes: {G_MEMSET, G_MEMSET_INLINE})
312 .unsupportedIf(Predicate: typeIs(TypeIdx: 0, TypesInit: p9))
313 .legalIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allPtrs), P1: typeInSet(TypeIdx: 1, TypesInit: allIntScalars)));
314
315 getActionDefinitionsBuilder(Opcode: G_ADDRSPACE_CAST)
316 .legalForCartesianProduct(Types0: allPtrs, Types1: allPtrs);
317
318 // Should we be legalizing bad scalar sizes like s5 here instead
319 // of handling them in the instruction selector?
320 getActionDefinitionsBuilder(Opcodes: {G_LOAD, G_STORE})
321 .unsupportedIf(Predicate: typeIs(TypeIdx: 1, TypesInit: p9))
322 .legalForCartesianProduct(Types0: allowedVectorTypes, Types1: allPtrs)
323 .legalForCartesianProduct(Types0: allPtrs, Types1: allPtrs)
324 .legalIf(Predicate: isScalar(TypeIdx: 0))
325 .custom();
326
327 getActionDefinitionsBuilder(Opcodes: {G_SMIN, G_SMAX, G_UMIN, G_UMAX, G_ABS,
328 G_BITREVERSE, G_SADDSAT, G_UADDSAT, G_SSUBSAT,
329 G_USUBSAT, G_SCMP, G_UCMP})
330 .legalFor(Types: allIntScalarsAndVectors)
331 .legalIf(Predicate: extendedScalarsAndVectors);
332
333 getActionDefinitionsBuilder(Opcodes: {G_SSHLSAT, G_USHLSAT}).lower();
334
335 getActionDefinitionsBuilder(Opcode: G_STRICT_FLDEXP)
336 .legalForCartesianProduct(Types0: allFloatScalarsAndVectors, Types1: allIntScalars);
337
338 getActionDefinitionsBuilder(Opcodes: {G_FPTOSI, G_FPTOUI})
339 .legalForCartesianProduct(Types0: allIntScalarsAndVectors,
340 Types1: allFloatScalarsAndVectors);
341
342 getActionDefinitionsBuilder(Opcodes: {G_FPTOSI_SAT, G_FPTOUI_SAT})
343 .legalForCartesianProduct(Types0: allIntScalarsAndVectors,
344 Types1: allFloatScalarsAndVectors);
345
346 getActionDefinitionsBuilder(Opcodes: {G_SITOFP, G_UITOFP})
347 .legalForCartesianProduct(Types0: allFloatScalarsAndVectors,
348 Types1: allScalarsAndVectors);
349
350 getActionDefinitionsBuilder(Opcode: G_CTPOP)
351 .legalForCartesianProduct(Types: allIntScalarsAndVectors)
352 .legalIf(Predicate: extendedScalarsAndVectorsProduct);
353
354 // Extensions.
355 getActionDefinitionsBuilder(Opcodes: {G_TRUNC, G_ZEXT, G_SEXT, G_ANYEXT})
356 .legalForCartesianProduct(Types: allScalarsAndVectors)
357 .legalIf(Predicate: extendedScalarsAndVectorsProduct);
358
359 // Lower G_SEXT_INREG to the canonical shl/ashr pair, which map to
360 // OpShiftLeftLogical + OpShiftRightArithmetic.
361 getActionDefinitionsBuilder(Opcode: G_SEXT_INREG).lower();
362
363 getActionDefinitionsBuilder(Opcode: G_PHI)
364 .legalFor(Types: allPtrsScalarsAndVectors)
365 .legalIf(Predicate: extendedPtrsScalarsAndVectors)
366 .moreElementsToNextPow2(TypeIdx: 0)
367 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
368 Mutation: LegalizeMutations::changeElementCountTo(
369 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)));
370
371 getActionDefinitionsBuilder(Opcode: G_BITCAST).legalIf(
372 Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allPtrsScalarsAndVectors),
373 P1: typeInSet(TypeIdx: 1, TypesInit: allPtrsScalarsAndVectors)));
374
375 getActionDefinitionsBuilder(Opcodes: {G_IMPLICIT_DEF, G_FREEZE})
376 .legalFor(Types: {s1, s128})
377 .legalFor(Types: allFloatAndIntScalarsAndPtrs)
378 .legalFor(Types: allowedVectorTypes)
379 .legalIf(Predicate: [](const LegalityQuery &Query) {
380 return Query.Types[0].isPointerVector();
381 })
382 .moreElementsToNextPow2(TypeIdx: 0)
383 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
384 Mutation: LegalizeMutations::changeElementCountTo(
385 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)));
386
387 getActionDefinitionsBuilder(Opcodes: {G_STACKSAVE, G_STACKRESTORE}).alwaysLegal();
388
389 getActionDefinitionsBuilder(Opcode: G_INTTOPTR)
390 .legalForCartesianProduct(Types0: allPtrs, Types1: allIntScalars)
391 .legalIf(
392 Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allPtrs), P1: typeOfExtendedScalars(TypeIdx: 1, IsExtendedInts)))
393 .legalIf(Predicate: [](const LegalityQuery &Query) {
394 const LLT DstTy = Query.Types[0];
395 const LLT SrcTy = Query.Types[1];
396 return DstTy.isPointerVector() && SrcTy.isVector() &&
397 !SrcTy.isPointer() &&
398 DstTy.getNumElements() == SrcTy.getNumElements();
399 });
400 getActionDefinitionsBuilder(Opcode: G_PTRTOINT)
401 .legalForCartesianProduct(Types0: allIntScalars, Types1: allPtrs)
402 .legalIf(
403 Predicate: all(P0: typeOfExtendedScalars(TypeIdx: 0, IsExtendedInts), P1: typeInSet(TypeIdx: 1, TypesInit: allPtrs)))
404 .legalIf(Predicate: [](const LegalityQuery &Query) {
405 const LLT DstTy = Query.Types[0];
406 const LLT SrcTy = Query.Types[1];
407 return SrcTy.isPointerVector() && DstTy.isVector() &&
408 !DstTy.isPointer() &&
409 DstTy.getNumElements() == SrcTy.getNumElements();
410 });
411 getActionDefinitionsBuilder(Opcode: G_PTR_ADD)
412 .legalForCartesianProduct(Types0: allPtrs, Types1: allIntScalars)
413 .legalIf(
414 Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allPtrs), P1: typeOfExtendedScalars(TypeIdx: 1, IsExtendedInts)));
415
416 getActionDefinitionsBuilder(Opcode: G_PTRMASK)
417 .legalForCartesianProduct(Types0: allPtrs, Types1: allIntScalars)
418 .legalIf(
419 Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allPtrs), P1: typeOfExtendedScalars(TypeIdx: 1, IsExtendedInts)))
420 .legalIf(Predicate: [](const LegalityQuery &Query) {
421 const LLT PtrTy = Query.Types[0];
422 const LLT MaskTy = Query.Types[1];
423 return PtrTy.isPointerVector() && MaskTy.isVector() &&
424 !MaskTy.isPointer() &&
425 PtrTy.getNumElements() == MaskTy.getNumElements();
426 });
427
428 // ST.canDirectlyComparePointers() for pointer args is supported in
429 // legalizeCustom().
430 getActionDefinitionsBuilder(Opcode: G_ICMP)
431 .unsupportedIf(Predicate: LegalityPredicates::any(
432 P0: all(P0: typeIs(TypeIdx: 0, TypesInit: p9), P1: typeInSet(TypeIdx: 1, TypesInit: allPtrs), args: typeIsNot(TypeIdx: 1, Type: p9)),
433 P1: all(P0: typeInSet(TypeIdx: 0, TypesInit: allPtrs), P1: typeIsNot(TypeIdx: 0, Type: p9), args: typeIs(TypeIdx: 1, TypesInit: p9))))
434 .legalIf(Predicate: [IsExtendedInts](const LegalityQuery &Query) {
435 const LLT Ty = Query.Types[1];
436 return IsExtendedInts && Ty.isValid() && !Ty.isPointerOrPointerVector();
437 })
438 .customIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allBoolScalarsAndVectors),
439 P1: typeInSet(TypeIdx: 1, TypesInit: allPtrsScalarsAndVectors)));
440
441 getActionDefinitionsBuilder(Opcode: G_FCMP).legalIf(
442 Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allBoolScalarsAndVectors),
443 P1: typeInSet(TypeIdx: 1, TypesInit: allFloatScalarsAndVectors)));
444
445 getActionDefinitionsBuilder(Opcodes: {G_ATOMICRMW_OR, G_ATOMICRMW_ADD, G_ATOMICRMW_AND,
446 G_ATOMICRMW_MAX, G_ATOMICRMW_MIN,
447 G_ATOMICRMW_SUB, G_ATOMICRMW_XOR,
448 G_ATOMICRMW_UMAX, G_ATOMICRMW_UMIN})
449 .legalForCartesianProduct(Types0: allIntScalars, Types1: allPtrs);
450
451 getActionDefinitionsBuilder(
452 Opcodes: {G_ATOMICRMW_FADD, G_ATOMICRMW_FSUB, G_ATOMICRMW_FMIN, G_ATOMICRMW_FMAX})
453 .legalForCartesianProduct(Types0: allFloatScalarsAndF16Vector2AndVector4s,
454 Types1: allPtrs);
455
456 getActionDefinitionsBuilder(Opcode: G_ATOMICRMW_XCHG)
457 .legalForCartesianProduct(Types0: allFloatAndIntScalarsAndPtrs, Types1: allPtrs);
458
459 getActionDefinitionsBuilder(Opcode: G_ATOMIC_CMPXCHG_WITH_SUCCESS).lower();
460 // TODO: add proper legalization rules.
461 getActionDefinitionsBuilder(Opcode: G_ATOMIC_CMPXCHG).alwaysLegal();
462
463 getActionDefinitionsBuilder(Opcodes: {G_UADDO, G_USUBO, G_UMULO, G_SMULO})
464 .alwaysLegal();
465
466 getActionDefinitionsBuilder(Opcodes: {G_SADDO, G_SSUBO}).lower();
467
468 // Lowering widens s64 to s128, which needs
469 // SPV_ALTERA_arbitrary_precision_integers. Mark s64 unsupported otherwise.
470 auto &MulFix = getActionDefinitionsBuilder(Opcodes: {G_SMULFIX, G_UMULFIX});
471 if (!HasArbitraryPrecisionInts)
472 MulFix.unsupportedFor(Types: {s64});
473 MulFix.lower();
474
475 getActionDefinitionsBuilder(Opcodes: {G_LROUND, G_LLROUND})
476 .legalForCartesianProduct(Types0: allFloatScalarsAndVectors,
477 Types1: allIntScalarsAndVectors);
478
479 // FP conversions.
480 getActionDefinitionsBuilder(Opcodes: {G_FPTRUNC, G_FPEXT})
481 .legalForCartesianProduct(Types: allFloatScalarsAndVectors);
482
483 // Pointer-handling.
484 getActionDefinitionsBuilder(Opcode: G_FRAME_INDEX).legalFor(Types: {p0});
485
486 getActionDefinitionsBuilder(Opcode: G_GLOBAL_VALUE).legalFor(Types: allPtrs);
487
488 // Control-flow. In some cases (e.g. constants) s1 may be promoted to s32.
489 getActionDefinitionsBuilder(Opcode: G_BR).alwaysLegal();
490 getActionDefinitionsBuilder(Opcode: G_BRCOND).legalFor(Types: {s1, s32});
491
492 getActionDefinitionsBuilder(Opcode: G_FFREXP).legalForCartesianProduct(
493 Types0: allFloatScalarsAndVectors, Types1: {s32, v2s32, v3s32, v4s32, v8s32, v16s32});
494
495 // TODO: Review the target OpenCL and GLSL Extended Instruction Set specs to
496 // tighten these requirements. Many of these math functions are only legal on
497 // specific bitwidths, so they are not selectable for
498 // allFloatScalarsAndVectors.
499 // clang-format off
500 getActionDefinitionsBuilder(Opcodes: {G_STRICT_FSQRT,
501 G_FPOW,
502 G_FEXP,
503 G_FMODF,
504 G_FSINCOS,
505 G_FEXP2,
506 G_FEXP10,
507 G_FLOG,
508 G_FLOG2,
509 G_FLOG10,
510 G_FABS,
511 G_FMINNUM,
512 G_FMAXNUM,
513 G_FCEIL,
514 G_FCOS,
515 G_FSIN,
516 G_FTAN,
517 G_FACOS,
518 G_FASIN,
519 G_FATAN,
520 G_FATAN2,
521 G_FCOSH,
522 G_FSINH,
523 G_FTANH,
524 G_FSQRT,
525 G_FFLOOR,
526 G_FRINT,
527 G_FNEARBYINT,
528 G_INTRINSIC_ROUND,
529 G_INTRINSIC_TRUNC,
530 G_FMINIMUM,
531 G_FMAXIMUM,
532 G_INTRINSIC_ROUNDEVEN})
533 .legalFor(Types: allFloatScalars)
534 .legalFor(Types: allowedFloatVectorTypes)
535 .moreElementsToNextPow2(TypeIdx: 0)
536 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
537 Mutation: LegalizeMutations::changeElementCountTo(
538 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)));
539 // clang-format on
540
541 getActionDefinitionsBuilder(Opcode: G_FCOPYSIGN)
542 .legalForCartesianProduct(Types0: allFloatScalarsAndVectors,
543 Types1: allFloatScalarsAndVectors);
544
545 getActionDefinitionsBuilder(Opcode: G_FPOWI).legalForCartesianProduct(
546 Types0: allFloatScalarsAndVectors, Types1: allIntScalarsAndVectors);
547
548 if (ST.canUseExtInstSet(E: SPIRV::InstructionSet::OpenCL_std)) {
549 getActionDefinitionsBuilder(
550 Opcodes: {G_CTTZ, G_CTTZ_ZERO_POISON, G_CTLZ, G_CTLZ_ZERO_POISON})
551 .legalForCartesianProduct(Types0: allIntScalarsAndVectors,
552 Types1: allIntScalarsAndVectors);
553
554 // Struct return types become a single scalar, so cannot easily legalize.
555 getActionDefinitionsBuilder(Opcodes: {G_SMULH, G_UMULH}).alwaysLegal();
556 }
557
558 getActionDefinitionsBuilder(Opcode: G_IS_FPCLASS).custom();
559
560 getActionDefinitionsBuilder(Opcodes: {G_INTRINSIC, G_INTRINSIC_CONVERGENT,
561 G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS})
562 .alwaysLegal();
563 getActionDefinitionsBuilder(Opcode: G_FENCE).alwaysLegal();
564 getActionDefinitionsBuilder(Opcodes: {G_TRAP, G_DEBUGTRAP, G_UBSANTRAP}).alwaysLegal();
565
566 verify(MII: *ST.getInstrInfo());
567}
568
569static bool legalizeExtractVectorElt(LegalizerHelper &Helper,
570 MachineInstr &MI) {
571 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
572 Register DstReg = MI.getOperand(i: 0).getReg();
573 Register SrcReg = MI.getOperand(i: 1).getReg();
574 Register IdxReg = MI.getOperand(i: 2).getReg();
575
576 MIRBuilder
577 .buildIntrinsic(ID: Intrinsic::spv_extractelt, Res: ArrayRef<Register>{DstReg})
578 .addUse(RegNo: SrcReg)
579 .addUse(RegNo: IdxReg);
580 MI.eraseFromParent();
581 return true;
582}
583
584static bool legalizeInsertVectorElt(LegalizerHelper &Helper, MachineInstr &MI) {
585 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
586 Register DstReg = MI.getOperand(i: 0).getReg();
587 Register SrcReg = MI.getOperand(i: 1).getReg();
588 Register ValReg = MI.getOperand(i: 2).getReg();
589 Register IdxReg = MI.getOperand(i: 3).getReg();
590
591 MIRBuilder
592 .buildIntrinsic(ID: Intrinsic::spv_insertelt, Res: ArrayRef<Register>{DstReg})
593 .addUse(RegNo: SrcReg)
594 .addUse(RegNo: ValReg)
595 .addUse(RegNo: IdxReg);
596 MI.eraseFromParent();
597 return true;
598}
599
600static Register convertPtrToInt(Register Reg, LLT ConvTy, SPIRVTypeInst SpvType,
601 LegalizerHelper &Helper,
602 MachineRegisterInfo &MRI,
603 SPIRVGlobalRegistry *GR) {
604 Register ConvReg = MRI.createGenericVirtualRegister(Ty: ConvTy);
605 MRI.setRegClass(Reg: ConvReg, RC: GR->getRegClass(SpvType));
606 GR->assignSPIRVTypeToVReg(Type: SpvType, VReg: ConvReg, MF: Helper.MIRBuilder.getMF());
607 Helper.MIRBuilder.buildInstr(Opcode: TargetOpcode::G_PTRTOINT)
608 .addDef(RegNo: ConvReg)
609 .addUse(RegNo: Reg);
610 return ConvReg;
611}
612
613static bool needsVectorLegalization(const LLT &Ty, const SPIRVSubtarget &ST) {
614 if (!Ty.isVector())
615 return false;
616 unsigned NumElements = Ty.getNumElements();
617 unsigned MaxVectorSize = ST.isShader() ? 4 : 16;
618 return (NumElements > 4 && !isPowerOf2_32(Value: NumElements)) ||
619 NumElements > MaxVectorSize;
620}
621
622static bool legalizeLoad(LegalizerHelper &Helper, MachineInstr &MI,
623 SPIRVGlobalRegistry *GR) {
624 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
625 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
626 Register DstReg = MI.getOperand(i: 0).getReg();
627 Register PtrReg = MI.getOperand(i: 1).getReg();
628 LLT DstTy = MRI.getType(Reg: DstReg);
629
630 if (!DstTy.isVector())
631 return true;
632
633 const SPIRVSubtarget &ST = MI.getMF()->getSubtarget<SPIRVSubtarget>();
634 if (!needsVectorLegalization(Ty: DstTy, ST))
635 return true;
636
637 SmallVector<Register, 8> SplitRegs;
638 LLT EltTy = DstTy.getElementType();
639 unsigned NumElts = DstTy.getNumElements();
640
641 LLT PtrTy = MRI.getType(Reg: PtrReg);
642 auto Zero = MIRBuilder.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: 0);
643
644 for (unsigned i = 0; i < NumElts; ++i) {
645 auto Idx = MIRBuilder.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: i);
646 Register EltPtr = MRI.createGenericVirtualRegister(Ty: PtrTy);
647
648 MIRBuilder.buildIntrinsic(ID: Intrinsic::spv_gep, Res: ArrayRef<Register>{EltPtr})
649 .addImm(Val: 1) // InBounds
650 .addUse(RegNo: PtrReg)
651 .addUse(RegNo: Zero.getReg(Idx: 0))
652 .addUse(RegNo: Idx.getReg(Idx: 0));
653
654 MachinePointerInfo EltPtrInfo;
655 Align EltAlign = Align(1);
656 if (!MI.memoperands_empty()) {
657 MachineMemOperand *MMO = *MI.memoperands_begin();
658 EltPtrInfo =
659 MMO->getPointerInfo().getWithOffset(O: i * EltTy.getSizeInBytes());
660 EltAlign = commonAlignment(A: MMO->getAlign(), Offset: i * EltTy.getSizeInBytes());
661 }
662
663 Register EltReg = MRI.createGenericVirtualRegister(Ty: EltTy);
664 MIRBuilder.buildLoad(Res: EltReg, Addr: EltPtr, PtrInfo: EltPtrInfo, Alignment: EltAlign);
665 SplitRegs.push_back(Elt: EltReg);
666 }
667
668 MIRBuilder.buildBuildVector(Res: DstReg, Ops: SplitRegs);
669 MI.eraseFromParent();
670 return true;
671}
672
673static bool legalizeStore(LegalizerHelper &Helper, MachineInstr &MI,
674 SPIRVGlobalRegistry *GR) {
675 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
676 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
677 Register ValReg = MI.getOperand(i: 0).getReg();
678 Register PtrReg = MI.getOperand(i: 1).getReg();
679 LLT ValTy = MRI.getType(Reg: ValReg);
680
681 assert(ValTy.isVector() && "Expected vector store");
682
683 SmallVector<Register, 8> SplitRegs;
684 LLT EltTy = ValTy.getElementType();
685 unsigned NumElts = ValTy.getNumElements();
686
687 for (unsigned i = 0; i < NumElts; ++i)
688 SplitRegs.push_back(Elt: MRI.createGenericVirtualRegister(Ty: EltTy));
689
690 MIRBuilder.buildUnmerge(Res: SplitRegs, Op: ValReg);
691
692 LLT PtrTy = MRI.getType(Reg: PtrReg);
693 auto Zero = MIRBuilder.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: 0);
694
695 for (unsigned i = 0; i < NumElts; ++i) {
696 auto Idx = MIRBuilder.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: i);
697 Register EltPtr = MRI.createGenericVirtualRegister(Ty: PtrTy);
698
699 MIRBuilder.buildIntrinsic(ID: Intrinsic::spv_gep, Res: ArrayRef<Register>{EltPtr})
700 .addImm(Val: 1) // InBounds
701 .addUse(RegNo: PtrReg)
702 .addUse(RegNo: Zero.getReg(Idx: 0))
703 .addUse(RegNo: Idx.getReg(Idx: 0));
704
705 MachinePointerInfo EltPtrInfo;
706 Align EltAlign = Align(1);
707 if (!MI.memoperands_empty()) {
708 MachineMemOperand *MMO = *MI.memoperands_begin();
709 EltPtrInfo =
710 MMO->getPointerInfo().getWithOffset(O: i * EltTy.getSizeInBytes());
711 EltAlign = commonAlignment(A: MMO->getAlign(), Offset: i * EltTy.getSizeInBytes());
712 }
713
714 MIRBuilder.buildStore(Val: SplitRegs[i], Addr: EltPtr, PtrInfo: EltPtrInfo, Alignment: EltAlign);
715 }
716
717 MI.eraseFromParent();
718 return true;
719}
720
721bool SPIRVLegalizerInfo::legalizeCustom(
722 LegalizerHelper &Helper, MachineInstr &MI,
723 LostDebugLocObserver &LocObserver) const {
724 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
725 switch (MI.getOpcode()) {
726 default:
727 // TODO: implement legalization for other opcodes.
728 return true;
729 case TargetOpcode::G_BITCAST:
730 return legalizeBitcast(Helper, MI);
731 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
732 return legalizeExtractVectorElt(Helper, MI);
733 case TargetOpcode::G_INSERT_VECTOR_ELT:
734 return legalizeInsertVectorElt(Helper, MI);
735 case TargetOpcode::G_INTRINSIC:
736 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
737 return legalizeIntrinsic(Helper, MI);
738 case TargetOpcode::G_IS_FPCLASS:
739 return legalizeIsFPClass(Helper, MI, LocObserver);
740 case TargetOpcode::G_ICMP: {
741 auto &Op0 = MI.getOperand(i: 2);
742 auto &Op1 = MI.getOperand(i: 3);
743 Register Reg0 = Op0.getReg();
744 Register Reg1 = Op1.getReg();
745 CmpInst::Predicate Cond =
746 static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
747 if ((!ST->canDirectlyComparePointers() ||
748 (Cond != CmpInst::ICMP_EQ && Cond != CmpInst::ICMP_NE)) &&
749 MRI.getType(Reg: Reg0).isPointer() && MRI.getType(Reg: Reg1).isPointer()) {
750 LLT ConvT = LLT::scalar(SizeInBits: ST->getPointerSize());
751 Type *LLVMTy = IntegerType::get(C&: MI.getMF()->getFunction().getContext(),
752 NumBits: ST->getPointerSize());
753 SPIRVTypeInst SpirvTy = GR->getOrCreateSPIRVType(
754 Type: LLVMTy, MIRBuilder&: Helper.MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
755 Op0.setReg(convertPtrToInt(Reg: Reg0, ConvTy: ConvT, SpvType: SpirvTy, Helper, MRI, GR));
756 Op1.setReg(convertPtrToInt(Reg: Reg1, ConvTy: ConvT, SpvType: SpirvTy, Helper, MRI, GR));
757 }
758 return true;
759 }
760 case TargetOpcode::G_LOAD:
761 return legalizeLoad(Helper, MI, GR);
762 case TargetOpcode::G_STORE:
763 return legalizeStore(Helper, MI, GR);
764 }
765}
766
767static MachineInstrBuilder
768createStackTemporaryForVector(LegalizerHelper &Helper, SPIRVGlobalRegistry *GR,
769 Register SrcReg, LLT SrcTy,
770 MachinePointerInfo &PtrInfo, Align &VecAlign) {
771 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
772 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
773
774 VecAlign = Helper.getStackTemporaryAlignment(Type: SrcTy);
775 auto StackTemp = Helper.createStackTemporary(
776 Bytes: TypeSize::getFixed(ExactSize: SrcTy.getSizeInBytes()), Alignment: VecAlign, PtrInfo);
777
778 // Set the type of StackTemp to a pointer to an array of the element type.
779 SPIRVTypeInst SpvSrcTy = GR->getSPIRVTypeForVReg(VReg: SrcReg);
780 SPIRVTypeInst EltSpvTy = GR->getScalarOrVectorComponentType(Type: SpvSrcTy);
781 const Type *LLVMEltTy = GR->getTypeForSPIRVType(Ty: EltSpvTy);
782 const Type *LLVMArrTy =
783 ArrayType::get(ElementType: const_cast<Type *>(LLVMEltTy), NumElements: SrcTy.getNumElements());
784 SPIRVTypeInst ArrSpvTy = GR->getOrCreateSPIRVType(
785 Type: LLVMArrTy, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
786 SPIRVTypeInst PtrToArrSpvTy = GR->getOrCreateSPIRVPointerType(
787 BaseType: ArrSpvTy, MIRBuilder, SC: SPIRV::StorageClass::Function);
788
789 Register StackReg = StackTemp.getReg(Idx: 0);
790 MRI.setRegClass(Reg: StackReg, RC: GR->getRegClass(SpvType: PtrToArrSpvTy));
791 GR->assignSPIRVTypeToVReg(Type: PtrToArrSpvTy, VReg: StackReg, MF: MIRBuilder.getMF());
792
793 return StackTemp;
794}
795
796static bool legalizeSpvBitcast(LegalizerHelper &Helper, MachineInstr &MI,
797 SPIRVGlobalRegistry *GR) {
798 LLVM_DEBUG(dbgs() << "Found a bitcast instruction\n");
799 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
800 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
801 const SPIRVSubtarget &ST = MI.getMF()->getSubtarget<SPIRVSubtarget>();
802
803 Register DstReg = MI.getOperand(i: 0).getReg();
804 Register SrcReg = MI.getOperand(i: 2).getReg();
805 LLT DstTy = MRI.getType(Reg: DstReg);
806 LLT SrcTy = MRI.getType(Reg: SrcReg);
807
808 // If an spv_bitcast needs to be legalized, we convert it to G_BITCAST to
809 // allow using the generic legalization rules.
810 if (needsVectorLegalization(Ty: DstTy, ST) ||
811 needsVectorLegalization(Ty: SrcTy, ST)) {
812 LLVM_DEBUG(dbgs() << "Replacing with a G_BITCAST\n");
813 MIRBuilder.buildBitcast(Dst: DstReg, Src: SrcReg);
814 MI.eraseFromParent();
815 }
816 return true;
817}
818
819static bool legalizeSpvInsertElt(LegalizerHelper &Helper, MachineInstr &MI,
820 SPIRVGlobalRegistry *GR) {
821 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
822 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
823 const SPIRVSubtarget &ST = MI.getMF()->getSubtarget<SPIRVSubtarget>();
824
825 Register DstReg = MI.getOperand(i: 0).getReg();
826 LLT DstTy = MRI.getType(Reg: DstReg);
827
828 if (needsVectorLegalization(Ty: DstTy, ST)) {
829 Register SrcReg = MI.getOperand(i: 2).getReg();
830 Register ValReg = MI.getOperand(i: 3).getReg();
831 LLT SrcTy = MRI.getType(Reg: SrcReg);
832 MachineOperand &IdxOperand = MI.getOperand(i: 4);
833
834 if (getImm(MO: IdxOperand, MRI: &MRI)) {
835 uint64_t IdxVal = foldImm(MO: IdxOperand, MRI: &MRI);
836 if (IdxVal < SrcTy.getNumElements()) {
837 SmallVector<Register, 8> Regs;
838 SPIRVTypeInst ElementType =
839 GR->getScalarOrVectorComponentType(Type: GR->getSPIRVTypeForVReg(VReg: DstReg));
840 LLT ElementLLTTy = GR->getRegType(SpvType: ElementType);
841 for (unsigned I = 0, E = SrcTy.getNumElements(); I < E; ++I) {
842 Register Reg = MRI.createGenericVirtualRegister(Ty: ElementLLTTy);
843 MRI.setRegClass(Reg, RC: GR->getRegClass(SpvType: ElementType));
844 GR->assignSPIRVTypeToVReg(Type: ElementType, VReg: Reg, MF: *MI.getMF());
845 Regs.push_back(Elt: Reg);
846 }
847 MIRBuilder.buildUnmerge(Res: Regs, Op: SrcReg);
848 Regs[IdxVal] = ValReg;
849 MIRBuilder.buildBuildVector(Res: DstReg, Ops: Regs);
850 MI.eraseFromParent();
851 return true;
852 }
853 }
854
855 LLT EltTy = SrcTy.getElementType();
856 Align VecAlign;
857 MachinePointerInfo PtrInfo;
858 auto StackTemp = createStackTemporaryForVector(Helper, GR, SrcReg, SrcTy,
859 PtrInfo, VecAlign);
860
861 MIRBuilder.buildStore(Val: SrcReg, Addr: StackTemp, PtrInfo, Alignment: VecAlign);
862
863 Register IdxReg = IdxOperand.getReg();
864 LLT PtrTy = MRI.getType(Reg: StackTemp.getReg(Idx: 0));
865 Register EltPtr = MRI.createGenericVirtualRegister(Ty: PtrTy);
866 auto Zero = MIRBuilder.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: 0);
867
868 MIRBuilder.buildIntrinsic(ID: Intrinsic::spv_gep, Res: ArrayRef<Register>{EltPtr})
869 .addImm(Val: 1) // InBounds
870 .addUse(RegNo: StackTemp.getReg(Idx: 0))
871 .addUse(RegNo: Zero.getReg(Idx: 0))
872 .addUse(RegNo: IdxReg);
873
874 MachinePointerInfo EltPtrInfo = MachinePointerInfo(PtrTy.getAddressSpace());
875 Align EltAlign = Helper.getStackTemporaryAlignment(Type: EltTy);
876 MIRBuilder.buildStore(Val: ValReg, Addr: EltPtr, PtrInfo: EltPtrInfo, Alignment: EltAlign);
877
878 MIRBuilder.buildLoad(Res: DstReg, Addr: StackTemp, PtrInfo, Alignment: VecAlign);
879 MI.eraseFromParent();
880 return true;
881 }
882 return true;
883}
884
885static bool legalizeSpvExtractElt(LegalizerHelper &Helper, MachineInstr &MI,
886 SPIRVGlobalRegistry *GR) {
887 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
888 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
889 const SPIRVSubtarget &ST = MI.getMF()->getSubtarget<SPIRVSubtarget>();
890
891 Register SrcReg = MI.getOperand(i: 2).getReg();
892 LLT SrcTy = MRI.getType(Reg: SrcReg);
893
894 if (needsVectorLegalization(Ty: SrcTy, ST)) {
895 Register DstReg = MI.getOperand(i: 0).getReg();
896 MachineOperand &IdxOperand = MI.getOperand(i: 3);
897
898 if (getImm(MO: IdxOperand, MRI: &MRI)) {
899 uint64_t IdxVal = foldImm(MO: IdxOperand, MRI: &MRI);
900 if (IdxVal < SrcTy.getNumElements()) {
901 LLT DstTy = MRI.getType(Reg: DstReg);
902 SmallVector<Register, 8> Regs;
903 SPIRVTypeInst DstSpvTy = GR->getSPIRVTypeForVReg(VReg: DstReg);
904 for (unsigned I = 0, E = SrcTy.getNumElements(); I < E; ++I) {
905 if (I == IdxVal) {
906 Regs.push_back(Elt: DstReg);
907 } else {
908 Register Reg = MRI.createGenericVirtualRegister(Ty: DstTy);
909 MRI.setRegClass(Reg, RC: GR->getRegClass(SpvType: DstSpvTy));
910 GR->assignSPIRVTypeToVReg(Type: DstSpvTy, VReg: Reg, MF: *MI.getMF());
911 Regs.push_back(Elt: Reg);
912 }
913 }
914 MIRBuilder.buildUnmerge(Res: Regs, Op: SrcReg);
915 MI.eraseFromParent();
916 return true;
917 }
918 }
919
920 LLT EltTy = SrcTy.getElementType();
921 Align VecAlign;
922 MachinePointerInfo PtrInfo;
923 auto StackTemp = createStackTemporaryForVector(Helper, GR, SrcReg, SrcTy,
924 PtrInfo, VecAlign);
925
926 MIRBuilder.buildStore(Val: SrcReg, Addr: StackTemp, PtrInfo, Alignment: VecAlign);
927
928 Register IdxReg = IdxOperand.getReg();
929 LLT PtrTy = MRI.getType(Reg: StackTemp.getReg(Idx: 0));
930 Register EltPtr = MRI.createGenericVirtualRegister(Ty: PtrTy);
931 auto Zero = MIRBuilder.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: 0);
932
933 MIRBuilder.buildIntrinsic(ID: Intrinsic::spv_gep, Res: ArrayRef<Register>{EltPtr})
934 .addImm(Val: 1) // InBounds
935 .addUse(RegNo: StackTemp.getReg(Idx: 0))
936 .addUse(RegNo: Zero.getReg(Idx: 0))
937 .addUse(RegNo: IdxReg);
938
939 MachinePointerInfo EltPtrInfo = MachinePointerInfo(PtrTy.getAddressSpace());
940 Align EltAlign = Helper.getStackTemporaryAlignment(Type: EltTy);
941 MIRBuilder.buildLoad(Res: DstReg, Addr: EltPtr, PtrInfo: EltPtrInfo, Alignment: EltAlign);
942
943 MI.eraseFromParent();
944 return true;
945 }
946 return true;
947}
948
949static bool legalizeSpvConstComposite(LegalizerHelper &Helper, MachineInstr &MI,
950 SPIRVGlobalRegistry *GR) {
951 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
952 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
953 const SPIRVSubtarget &ST = MI.getMF()->getSubtarget<SPIRVSubtarget>();
954
955 Register DstReg = MI.getOperand(i: 0).getReg();
956 LLT DstTy = MRI.getType(Reg: DstReg);
957
958 if (!needsVectorLegalization(Ty: DstTy, ST))
959 return true;
960
961 SmallVector<Register, 8> SrcRegs;
962 if (MI.getNumOperands() == 2) {
963 // The "null" case: no values are attached.
964 LLT EltTy = DstTy.getElementType();
965 auto Zero = MIRBuilder.buildConstant(Res: EltTy, Val: 0);
966 SPIRVTypeInst SpvDstTy = GR->getSPIRVTypeForVReg(VReg: DstReg);
967 SPIRVTypeInst SpvEltTy = GR->getScalarOrVectorComponentType(Type: SpvDstTy);
968 GR->assignSPIRVTypeToVReg(Type: SpvEltTy, VReg: Zero.getReg(Idx: 0), MF: MIRBuilder.getMF());
969 for (unsigned i = 0; i < DstTy.getNumElements(); ++i)
970 SrcRegs.push_back(Elt: Zero.getReg(Idx: 0));
971 } else {
972 for (unsigned i = 2; i < MI.getNumOperands(); ++i) {
973 SrcRegs.push_back(Elt: MI.getOperand(i).getReg());
974 }
975 }
976 MIRBuilder.buildBuildVector(Res: DstReg, Ops: SrcRegs);
977 MI.eraseFromParent();
978 return true;
979}
980
981bool SPIRVLegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
982 MachineInstr &MI) const {
983 LLVM_DEBUG(dbgs() << "legalizeIntrinsic: " << MI);
984 auto IntrinsicID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
985 switch (IntrinsicID) {
986 case Intrinsic::spv_bitcast:
987 return legalizeSpvBitcast(Helper, MI, GR);
988 case Intrinsic::spv_insertelt:
989 return legalizeSpvInsertElt(Helper, MI, GR);
990 case Intrinsic::spv_extractelt:
991 return legalizeSpvExtractElt(Helper, MI, GR);
992 case Intrinsic::spv_const_composite:
993 return legalizeSpvConstComposite(Helper, MI, GR);
994 }
995 return true;
996}
997
998bool SPIRVLegalizerInfo::legalizeBitcast(LegalizerHelper &Helper,
999 MachineInstr &MI) const {
1000 // Once the G_BITCAST is using vectors that are allowed, we turn it back into
1001 // an spv_bitcast to avoid verifier problems when the register types are the
1002 // same for the source and the result. Note that the SPIR-V types associated
1003 // with the bitcast can be different even if the register types are the same.
1004 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1005 Register DstReg = MI.getOperand(i: 0).getReg();
1006 Register SrcReg = MI.getOperand(i: 1).getReg();
1007 SmallVector<Register, 1> DstRegs = {DstReg};
1008 MIRBuilder.buildIntrinsic(ID: Intrinsic::spv_bitcast, Res: DstRegs).addUse(RegNo: SrcReg);
1009 MI.eraseFromParent();
1010 return true;
1011}
1012
1013// Note this code was copied from LegalizerHelper::lowerISFPCLASS and adjusted
1014// to ensure that all instructions created during the lowering have SPIR-V types
1015// assigned to them.
1016bool SPIRVLegalizerInfo::legalizeIsFPClass(
1017 LegalizerHelper &Helper, MachineInstr &MI,
1018 LostDebugLocObserver &LocObserver) const {
1019 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
1020 FPClassTest Mask = static_cast<FPClassTest>(MI.getOperand(i: 2).getImm());
1021
1022 auto &MIRBuilder = Helper.MIRBuilder;
1023 auto &MF = MIRBuilder.getMF();
1024 MachineRegisterInfo &MRI = MF.getRegInfo();
1025
1026 Type *LLVMDstTy =
1027 IntegerType::get(C&: MIRBuilder.getContext(), NumBits: DstTy.getScalarSizeInBits());
1028 if (DstTy.isVector())
1029 LLVMDstTy = VectorType::get(ElementType: LLVMDstTy, EC: DstTy.getElementCount());
1030 SPIRVTypeInst SPIRVDstTy = GR->getOrCreateSPIRVType(
1031 Type: LLVMDstTy, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite,
1032 /*EmitIR*/ true);
1033
1034 unsigned BitSize = SrcTy.getScalarSizeInBits();
1035 const fltSemantics &Semantics = getFltSemanticForLLT(Ty: SrcTy.getScalarType());
1036
1037 LLT IntTy = LLT::scalar(SizeInBits: BitSize);
1038 Type *LLVMIntTy = IntegerType::get(C&: MIRBuilder.getContext(), NumBits: BitSize);
1039 if (SrcTy.isVector()) {
1040 IntTy = LLT::vector(EC: SrcTy.getElementCount(), ScalarTy: IntTy);
1041 LLVMIntTy = VectorType::get(ElementType: LLVMIntTy, EC: SrcTy.getElementCount());
1042 }
1043 SPIRVTypeInst SPIRVIntTy = GR->getOrCreateSPIRVType(
1044 Type: LLVMIntTy, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite,
1045 /*EmitIR*/ true);
1046
1047 // Clang doesn't support capture of structured bindings:
1048 LLT DstTyCopy = DstTy;
1049 const auto assignSPIRVTy = [&](MachineInstrBuilder &&MI) {
1050 // Assign this MI's (assumed only) destination to one of the two types we
1051 // expect: either the G_IS_FPCLASS's destination type, or the integer type
1052 // bitcast from the source type.
1053 LLT MITy = MRI.getType(Reg: MI.getReg(Idx: 0));
1054 assert((MITy == IntTy || MITy == DstTyCopy) &&
1055 "Unexpected LLT type while lowering G_IS_FPCLASS");
1056 SPIRVTypeInst SPVTy = MITy == IntTy ? SPIRVIntTy : SPIRVDstTy;
1057 GR->assignSPIRVTypeToVReg(Type: SPVTy, VReg: MI.getReg(Idx: 0), MF);
1058 return MI;
1059 };
1060
1061 // Helper to build and assign a constant in one go
1062 const auto buildSPIRVConstant = [&](LLT Ty, auto &&C) -> MachineInstrBuilder {
1063 if (!Ty.isFixedVector())
1064 return assignSPIRVTy(MIRBuilder.buildConstant(Ty, C));
1065 auto ScalarC = MIRBuilder.buildConstant(Ty.getScalarType(), C);
1066 assert((Ty == IntTy || Ty == DstTyCopy) &&
1067 "Unexpected LLT type while lowering constant for G_IS_FPCLASS");
1068 SPIRVTypeInst VecEltTy = GR->getOrCreateSPIRVType(
1069 Type: (Ty == IntTy ? LLVMIntTy : LLVMDstTy)->getScalarType(), MIRBuilder,
1070 AQ: SPIRV::AccessQualifier::ReadWrite,
1071 /*EmitIR*/ true);
1072 GR->assignSPIRVTypeToVReg(Type: VecEltTy, VReg: ScalarC.getReg(0), MF);
1073 return assignSPIRVTy(MIRBuilder.buildSplatBuildVector(Res: Ty, Src: ScalarC));
1074 };
1075
1076 if (Mask == fcNone) {
1077 MIRBuilder.buildCopy(Res: DstReg, Op: buildSPIRVConstant(DstTy, 0));
1078 MI.eraseFromParent();
1079 return true;
1080 }
1081 if (Mask == fcAllFlags) {
1082 MIRBuilder.buildCopy(Res: DstReg, Op: buildSPIRVConstant(DstTy, 1));
1083 MI.eraseFromParent();
1084 return true;
1085 }
1086
1087 // Note that rather than creating a COPY here (between a floating-point and
1088 // integer type of the same size) we create a SPIR-V bitcast immediately. We
1089 // can't create a G_BITCAST because the LLTs are the same, and we can't seem
1090 // to correctly lower COPYs to SPIR-V bitcasts at this moment.
1091 Register ResVReg = MRI.createGenericVirtualRegister(Ty: IntTy);
1092 MRI.setRegClass(Reg: ResVReg, RC: GR->getRegClass(SpvType: SPIRVIntTy));
1093 GR->assignSPIRVTypeToVReg(Type: SPIRVIntTy, VReg: ResVReg, MF: Helper.MIRBuilder.getMF());
1094 auto AsInt = MIRBuilder.buildInstr(Opcode: SPIRV::OpBitcast)
1095 .addDef(RegNo: ResVReg)
1096 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: SPIRVIntTy))
1097 .addUse(RegNo: SrcReg);
1098 AsInt = assignSPIRVTy(std::move(AsInt));
1099
1100 // Various masks.
1101 APInt SignBit = APInt::getSignMask(BitWidth: BitSize);
1102 APInt ValueMask = APInt::getSignedMaxValue(numBits: BitSize); // All bits but sign.
1103 APInt Inf = APFloat::getInf(Sem: Semantics).bitcastToAPInt(); // Exp and int bit.
1104 APInt ExpMask = Inf;
1105 APInt AllOneMantissa = APFloat::getLargest(Sem: Semantics).bitcastToAPInt() & ~Inf;
1106 APInt QNaNBitMask =
1107 APInt::getOneBitSet(numBits: BitSize, BitNo: AllOneMantissa.getActiveBits() - 1);
1108 APInt InversionMask = APInt::getAllOnes(numBits: DstTy.getScalarSizeInBits());
1109
1110 auto SignBitC = buildSPIRVConstant(IntTy, SignBit);
1111 auto ValueMaskC = buildSPIRVConstant(IntTy, ValueMask);
1112 auto InfC = buildSPIRVConstant(IntTy, Inf);
1113 auto ExpMaskC = buildSPIRVConstant(IntTy, ExpMask);
1114 auto ZeroC = buildSPIRVConstant(IntTy, 0);
1115
1116 auto Abs = assignSPIRVTy(MIRBuilder.buildAnd(Dst: IntTy, Src0: AsInt, Src1: ValueMaskC));
1117 auto Sign = assignSPIRVTy(
1118 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_NE, Res: DstTy, Op0: AsInt, Op1: Abs));
1119
1120 auto Res = buildSPIRVConstant(DstTy, 0);
1121
1122 const auto appendToRes = [&](MachineInstrBuilder &&ToAppend) {
1123 Res = assignSPIRVTy(
1124 MIRBuilder.buildOr(Dst: DstTyCopy, Src0: Res, Src1: assignSPIRVTy(std::move(ToAppend))));
1125 };
1126
1127 // Tests that involve more than one class should be processed first.
1128 if ((Mask & fcFinite) == fcFinite) {
1129 // finite(V) ==> abs(V) u< exp_mask
1130 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: Abs,
1131 Op1: ExpMaskC));
1132 Mask &= ~fcFinite;
1133 } else if ((Mask & fcFinite) == fcPosFinite) {
1134 // finite(V) && V > 0 ==> V u< exp_mask
1135 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: AsInt,
1136 Op1: ExpMaskC));
1137 Mask &= ~fcPosFinite;
1138 } else if ((Mask & fcFinite) == fcNegFinite) {
1139 // finite(V) && V < 0 ==> abs(V) u< exp_mask && signbit == 1
1140 auto Cmp = assignSPIRVTy(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT,
1141 Res: DstTy, Op0: Abs, Op1: ExpMaskC));
1142 appendToRes(MIRBuilder.buildAnd(Dst: DstTy, Src0: Cmp, Src1: Sign));
1143 Mask &= ~fcNegFinite;
1144 }
1145
1146 if (FPClassTest PartialCheck = Mask & (fcZero | fcSubnormal)) {
1147 // fcZero | fcSubnormal => test all exponent bits are 0
1148 // TODO: Handle sign bit specific cases
1149 // TODO: Handle inverted case
1150 if (PartialCheck == (fcZero | fcSubnormal)) {
1151 auto ExpBits = assignSPIRVTy(MIRBuilder.buildAnd(Dst: IntTy, Src0: AsInt, Src1: ExpMaskC));
1152 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
1153 Op0: ExpBits, Op1: ZeroC));
1154 Mask &= ~PartialCheck;
1155 }
1156 }
1157
1158 // Check for individual classes.
1159 if (FPClassTest PartialCheck = Mask & fcZero) {
1160 if (PartialCheck == fcPosZero)
1161 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
1162 Op0: AsInt, Op1: ZeroC));
1163 else if (PartialCheck == fcZero)
1164 appendToRes(
1165 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy, Op0: Abs, Op1: ZeroC));
1166 else // fcNegZero
1167 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
1168 Op0: AsInt, Op1: SignBitC));
1169 }
1170
1171 if (FPClassTest PartialCheck = Mask & fcSubnormal) {
1172 // issubnormal(V) ==> unsigned(abs(V) - 1) u< (all mantissa bits set)
1173 // issubnormal(V) && V>0 ==> unsigned(V - 1) u< (all mantissa bits set)
1174 auto V = (PartialCheck == fcPosSubnormal) ? AsInt : Abs;
1175 auto OneC = buildSPIRVConstant(IntTy, 1);
1176 auto VMinusOne = MIRBuilder.buildSub(Dst: IntTy, Src0: V, Src1: OneC);
1177 auto SubnormalRes = assignSPIRVTy(
1178 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: VMinusOne,
1179 Op1: buildSPIRVConstant(IntTy, AllOneMantissa)));
1180 if (PartialCheck == fcNegSubnormal)
1181 SubnormalRes = MIRBuilder.buildAnd(Dst: DstTy, Src0: SubnormalRes, Src1: Sign);
1182 appendToRes(std::move(SubnormalRes));
1183 }
1184
1185 if (FPClassTest PartialCheck = Mask & fcInf) {
1186 if (PartialCheck == fcPosInf)
1187 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
1188 Op0: AsInt, Op1: InfC));
1189 else if (PartialCheck == fcInf)
1190 appendToRes(
1191 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy, Op0: Abs, Op1: InfC));
1192 else { // fcNegInf
1193 APInt NegInf = APFloat::getInf(Sem: Semantics, Negative: true).bitcastToAPInt();
1194 auto NegInfC = buildSPIRVConstant(IntTy, NegInf);
1195 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
1196 Op0: AsInt, Op1: NegInfC));
1197 }
1198 }
1199
1200 if (FPClassTest PartialCheck = Mask & fcNan) {
1201 auto InfWithQnanBitC =
1202 buildSPIRVConstant(IntTy, std::move(Inf) | QNaNBitMask);
1203 if (PartialCheck == fcNan) {
1204 // isnan(V) ==> abs(V) u> int(inf)
1205 appendToRes(
1206 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_UGT, Res: DstTy, Op0: Abs, Op1: InfC));
1207 } else if (PartialCheck == fcQNan) {
1208 // isquiet(V) ==> abs(V) u>= (unsigned(Inf) | quiet_bit)
1209 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_UGE, Res: DstTy, Op0: Abs,
1210 Op1: InfWithQnanBitC));
1211 } else { // fcSNan
1212 // issignaling(V) ==> abs(V) u> unsigned(Inf) &&
1213 // abs(V) u< (unsigned(Inf) | quiet_bit)
1214 auto IsNan = assignSPIRVTy(
1215 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_UGT, Res: DstTy, Op0: Abs, Op1: InfC));
1216 auto IsNotQnan = assignSPIRVTy(MIRBuilder.buildICmp(
1217 Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: Abs, Op1: InfWithQnanBitC));
1218 appendToRes(MIRBuilder.buildAnd(Dst: DstTy, Src0: IsNan, Src1: IsNotQnan));
1219 }
1220 }
1221
1222 if (FPClassTest PartialCheck = Mask & fcNormal) {
1223 // isnormal(V) ==> (0 u< exp u< max_exp) ==> (unsigned(exp-1) u<
1224 // (max_exp-1))
1225 APInt ExpLSB = ExpMask & ~(ExpMask.shl(shiftAmt: 1));
1226 auto ExpMinusOne = assignSPIRVTy(
1227 MIRBuilder.buildSub(Dst: IntTy, Src0: Abs, Src1: buildSPIRVConstant(IntTy, ExpLSB)));
1228 APInt MaxExpMinusOne = std::move(ExpMask) - ExpLSB;
1229 auto NormalRes = assignSPIRVTy(
1230 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: ExpMinusOne,
1231 Op1: buildSPIRVConstant(IntTy, MaxExpMinusOne)));
1232 if (PartialCheck == fcNegNormal)
1233 NormalRes = MIRBuilder.buildAnd(Dst: DstTy, Src0: NormalRes, Src1: Sign);
1234 else if (PartialCheck == fcPosNormal) {
1235 auto PosSign = assignSPIRVTy(MIRBuilder.buildXor(
1236 Dst: DstTy, Src0: Sign, Src1: buildSPIRVConstant(DstTy, InversionMask)));
1237 NormalRes = MIRBuilder.buildAnd(Dst: DstTy, Src0: NormalRes, Src1: PosSign);
1238 }
1239 appendToRes(std::move(NormalRes));
1240 }
1241
1242 MIRBuilder.buildCopy(Res: DstReg, Op: Res);
1243 MI.eraseFromParent();
1244 return true;
1245}
1246