1//===- SPIRVLegalizerInfo.cpp --- SPIR-V Legalization Rules ------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the targeting of the Machinelegalizer class for SPIR-V.
10//
11//===----------------------------------------------------------------------===//
12
13#include "SPIRVLegalizerInfo.h"
14#include "SPIRV.h"
15#include "SPIRVGlobalRegistry.h"
16#include "SPIRVSubtarget.h"
17#include "SPIRVUtils.h"
18#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
19#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
20#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
21#include "llvm/CodeGen/MachineInstr.h"
22#include "llvm/CodeGen/MachineRegisterInfo.h"
23#include "llvm/CodeGen/TargetOpcodes.h"
24#include "llvm/IR/IntrinsicsSPIRV.h"
25#include "llvm/Support/Debug.h"
26#include "llvm/Support/MathExtras.h"
27#include "llvm/Support/SPIRVAddrSpace.h"
28
29using namespace llvm;
30using namespace llvm::LegalizeActions;
31using namespace llvm::LegalityPredicates;
32
33#define DEBUG_TYPE "spirv-legalizer"
34
35LegalityPredicate typeOfExtendedScalars(unsigned TypeIdx, bool IsExtendedInts) {
36 return [IsExtendedInts, TypeIdx](const LegalityQuery &Query) {
37 const LLT Ty = Query.Types[TypeIdx];
38 return IsExtendedInts && Ty.isValid() && Ty.isScalar();
39 };
40}
41
42LegalityPredicate typeOfLongVectors(unsigned TypeIdx, bool IsLongVecs) {
43 return [TypeIdx, IsLongVecs](const LegalityQuery &Query) {
44 const LLT Ty = Query.Types[TypeIdx];
45 return IsLongVecs && Ty.isValid() && Ty.isVector();
46 };
47}
48
49static bool needsVectorLegalization(const LLT &Ty, const SPIRVSubtarget &ST) {
50 if (!Ty.isVector() ||
51 ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_long_vector))
52 return false;
53 unsigned NumElements = Ty.getNumElements();
54 unsigned MaxVectorSize = ST.isShader() ? 4 : 16;
55 return (NumElements > 4 && !isPowerOf2_32(Value: NumElements)) ||
56 NumElements > MaxVectorSize;
57}
58
59SPIRVLegalizerInfo::SPIRVLegalizerInfo(const SPIRVSubtarget &ST) {
60 using namespace TargetOpcode;
61
62 this->ST = &ST;
63 GR = ST.getSPIRVGlobalRegistry();
64
65 const LLT s1 = LLT::scalar(SizeInBits: 1);
66 const LLT s8 = LLT::scalar(SizeInBits: 8);
67 const LLT s16 = LLT::scalar(SizeInBits: 16);
68 const LLT s32 = LLT::scalar(SizeInBits: 32);
69 const LLT s64 = LLT::scalar(SizeInBits: 64);
70 const LLT s128 = LLT::scalar(SizeInBits: 128);
71
72 const LLT v16s64 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 64);
73 const LLT v16s32 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 32);
74 const LLT v16s16 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 16);
75 const LLT v16s8 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 8);
76 const LLT v16s1 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 1);
77
78 const LLT v8s64 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 64);
79 const LLT v8s32 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 32);
80 const LLT v8s16 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16);
81 const LLT v8s8 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 8);
82 const LLT v8s1 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 1);
83
84 const LLT v4s64 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 64);
85 const LLT v4s32 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32);
86 const LLT v4s16 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 16);
87 const LLT v4s8 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 8);
88 const LLT v4s1 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 1);
89
90 const LLT v3s64 = LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 64);
91 const LLT v3s32 = LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 32);
92 const LLT v3s16 = LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 16);
93 const LLT v3s8 = LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 8);
94 const LLT v3s1 = LLT::fixed_vector(NumElements: 3, ScalarSizeInBits: 1);
95
96 const LLT v2s64 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64);
97 const LLT v2s32 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32);
98 const LLT v2s16 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 16);
99 const LLT v2s8 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 8);
100 const LLT v2s1 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 1);
101
102 const unsigned PSize = ST.getPointerSize();
103 const LLT p0 = LLT::pointer(AddressSpace: SPIRVAS::Function, SizeInBits: PSize);
104 const LLT p1 = LLT::pointer(AddressSpace: SPIRVAS::CrossWorkgroup, SizeInBits: PSize);
105 const LLT p2 = LLT::pointer(AddressSpace: SPIRVAS::UniformConstant, SizeInBits: PSize);
106 const LLT p3 = LLT::pointer(AddressSpace: SPIRVAS::Workgroup, SizeInBits: PSize);
107 const LLT p4 = LLT::pointer(AddressSpace: SPIRVAS::Generic, SizeInBits: PSize);
108 const LLT p5 = LLT::pointer(AddressSpace: SPIRVAS::DeviceOnlyINTEL, SizeInBits: PSize);
109 const LLT p6 = LLT::pointer(AddressSpace: SPIRVAS::HostOnlyINTEL, SizeInBits: PSize);
110 const LLT p7 = LLT::pointer(AddressSpace: SPIRVAS::Input, SizeInBits: PSize);
111 const LLT p8 = LLT::pointer(AddressSpace: SPIRVAS::Output, SizeInBits: PSize);
112 const LLT p9 = LLT::pointer(AddressSpace: SPIRVAS::CodeSectionINTEL, SizeInBits: PSize);
113 const LLT p10 = LLT::pointer(AddressSpace: SPIRVAS::Private, SizeInBits: PSize);
114 const LLT p11 = LLT::pointer(AddressSpace: SPIRVAS::StorageBuffer, SizeInBits: PSize);
115 const LLT p12 = LLT::pointer(AddressSpace: SPIRVAS::Uniform, SizeInBits: PSize);
116 const LLT p13 = LLT::pointer(AddressSpace: SPIRVAS::PushConstant, SizeInBits: PSize);
117
118 // TODO: remove copy-pasting here by using concatenation in some way.
119 auto allPtrsScalarsAndVectors = {
120 p0, p1, p2, p3, p4, p5, p6, p7, p8,
121 p9, p10, p11, p12, p13, s1, s8, s16, s32,
122 s64, s128, v2s1, v2s8, v2s16, v2s32, v2s64, v3s1, v3s8,
123 v3s16, v3s32, v3s64, v4s1, v4s8, v4s16, v4s32, v4s64, v8s1,
124 v8s8, v8s16, v8s32, v8s64, v16s1, v16s8, v16s16, v16s32, v16s64};
125
126 auto allVectors = {v2s1, v2s8, v2s16, v2s32, v2s64, v3s1, v3s8,
127 v3s16, v3s32, v3s64, v4s1, v4s8, v4s16, v4s32,
128 v4s64, v8s1, v8s8, v8s16, v8s32, v8s64, v16s1,
129 v16s8, v16s16, v16s32, v16s64};
130
131 auto allShaderVectors = {v2s1, v2s8, v2s16, v2s32, v2s64,
132 v3s1, v3s8, v3s16, v3s32, v3s64,
133 v4s1, v4s8, v4s16, v4s32, v4s64};
134
135 auto allScalars = {s1, s8, s16, s32, s64};
136
137 auto allScalarsAndVectors = {
138 s1, s8, s16, s32, s64, s128, v2s1, v2s8,
139 v2s16, v2s32, v2s64, v3s1, v3s8, v3s16, v3s32, v3s64,
140 v4s1, v4s8, v4s16, v4s32, v4s64, v8s1, v8s8, v8s16,
141 v8s32, v8s64, v16s1, v16s8, v16s16, v16s32, v16s64};
142
143 auto allShaderScalarsAndVectors = {
144 s1, s8, s16, s32, s64, s128, v2s1, v2s8, v2s16, v2s32, v2s64,
145 v3s1, v3s8, v3s16, v3s32, v3s64, v4s1, v4s8, v4s16, v4s32, v4s64};
146
147 auto &allowedScalarsAndVectors =
148 ST.isShader() ? allShaderScalarsAndVectors : allScalarsAndVectors;
149
150 auto allIntScalarsAndVectors = {
151 s8, s16, s32, s64, s128, v2s8, v2s16, v2s32, v2s64,
152 v3s8, v3s16, v3s32, v3s64, v4s8, v4s16, v4s32, v4s64, v8s8,
153 v8s16, v8s32, v8s64, v16s8, v16s16, v16s32, v16s64};
154
155 auto allBoolScalarsAndVectors = {s1, v2s1, v3s1, v4s1, v8s1, v16s1};
156 auto allBoolVectors = {v2s1, v3s1, v4s1, v8s1, v16s1};
157
158 auto allIntScalars = {s8, s16, s32, s64, s128};
159
160 auto allShaderIntVectors = {v2s8, v2s16, v2s32, v2s64, v3s8, v3s16,
161 v3s32, v3s64, v4s8, v4s16, v4s32, v4s64};
162
163 auto allIntVectors = {v2s8, v2s16, v2s32, v2s64, v3s8, v3s16, v3s32,
164 v3s64, v4s8, v4s16, v4s32, v4s64, v8s8, v8s16,
165 v8s32, v8s64, v16s8, v16s16, v16s32, v16s64};
166
167 auto &allowedIntVectorTypes =
168 ST.isShader() ? allShaderIntVectors : allIntVectors;
169
170 auto allFloatScalarsAndF16Vector2AndVector4s = {s16, s32, s64, v2s16, v4s16};
171
172 auto allFloatScalars = {s16, s32, s64};
173
174 auto allFloatScalarsAndVectors = {
175 s16, s32, s64, v2s16, v2s32, v2s64, v3s16, v3s32, v3s64,
176 v4s16, v4s32, v4s64, v8s16, v8s32, v8s64, v16s16, v16s32, v16s64};
177
178 auto allShaderFloatVectors = {v2s16, v2s32, v2s64, v3s16, v3s32,
179 v3s64, v4s16, v4s32, v4s64};
180
181 auto allFloatVectors = {v2s16, v2s32, v2s64, v3s16, v3s32,
182 v3s64, v4s16, v4s32, v4s64, v8s16,
183 v8s32, v8s64, v16s16, v16s32, v16s64};
184
185 auto &allowedFloatVectorTypes =
186 ST.isShader() ? allShaderFloatVectors : allFloatVectors;
187
188 auto allFloatAndIntScalarsAndPtrs = {s8, s16, s32, s64, p0, p1,
189 p2, p3, p4, p5, p6, p7,
190 p8, p9, p10, p11, p12, p13};
191
192 auto allPtrs = {p0, p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13};
193
194 auto &allowedVectorTypes = ST.isShader() ? allShaderVectors : allVectors;
195
196 bool HasArbitraryPrecisionInts = ST.canUseExtension(
197 E: SPIRV::Extension::SPV_ALTERA_arbitrary_precision_integers);
198 bool IsExtendedInts =
199 HasArbitraryPrecisionInts ||
200 ST.canUseExtension(E: SPIRV::Extension::SPV_KHR_bit_instructions) ||
201 ST.canUseExtension(E: SPIRV::Extension::SPV_INTEL_int4);
202 bool IsLongVecs = ST.canUseExtension(E: SPIRV::Extension::SPV_EXT_long_vector);
203 auto ExtendedIntScalarsAndVectors =
204 [IsExtendedInts](const LegalityQuery &Query) {
205 const LLT Ty = Query.Types[0];
206 return IsExtendedInts && Ty.isValid() &&
207 !Ty.isPointerOrPointerVector() && Ty.getScalarSizeInBits() > 1;
208 };
209 auto ExtendedScalarsAndVectorsProduct = [IsExtendedInts](
210 const LegalityQuery &Query) {
211 const LLT Ty1 = Query.Types[0], Ty2 = Query.Types[1];
212 return IsExtendedInts && Ty1.isValid() && Ty2.isValid() &&
213 !Ty1.isPointerOrPointerVector() && !Ty2.isPointerOrPointerVector();
214 };
215 auto ExtendedPtrsScalarsAndVectors =
216 [IsExtendedInts](const LegalityQuery &Query) {
217 const LLT Ty = Query.Types[0];
218 return IsExtendedInts && Ty.isValid();
219 };
220
221 // The universal validation rules in the SPIR-V specification state that
222 // vector sizes are typically limited to 2, 3, or 4. However, larger vector
223 // sizes (8 and 16) are enabled when the Kernel capability is present. For
224 // shader execution models, vector sizes are strictly limited to 4. In
225 // non-shader contexts, vector sizes of 8 and 16 are also permitted, but
226 // arbitrary sizes (e.g., 6 or 11) are not.
227 uint32_t MaxVectorSize = ST.isShader() ? 4 : 16;
228 LLVM_DEBUG(dbgs() << "MaxVectorSize: " << MaxVectorSize << "\n");
229
230 for (auto Opc : getTypeFoldingSupportedOpcodes()) {
231 switch (Opc) {
232 case G_EXTRACT_VECTOR_ELT:
233 case G_UREM:
234 case G_SREM:
235 case G_UDIV:
236 case G_SDIV:
237 case G_FREM:
238 case G_SELECT:
239 break;
240 default:
241 getActionDefinitionsBuilder(Opcode: Opc)
242 .customFor(Types: allScalars)
243 .customFor(Types: allowedVectorTypes)
244 .customIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
245 .moreElementsToNextPow2(TypeIdx: 0)
246 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
247 Mutation: LegalizeMutations::changeElementCountTo(
248 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
249 .custom();
250 break;
251 }
252 }
253
254 getActionDefinitionsBuilder(Opcodes: {G_UREM, G_SREM, G_SDIV, G_UDIV, G_FREM})
255 .customFor(Types: allScalars)
256 .customFor(Types: allowedVectorTypes)
257 .customIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
258 .scalarizeIf(Predicate: numElementsNotPow2(TypeIdx: 0), TypeIdx: 0)
259 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
260 Mutation: LegalizeMutations::changeElementCountTo(
261 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
262 .custom();
263
264 getActionDefinitionsBuilder(Opcode: G_SELECT)
265 .customFor(Types: allScalars)
266 .customFor(Types: allowedVectorTypes)
267 .customIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
268 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
269 Mutation: LegalizeMutations::changeElementCountTo(
270 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
271 .custom();
272
273 getActionDefinitionsBuilder(Opcodes: {G_FMA, G_STRICT_FMA})
274 .legalFor(Types: allScalars)
275 .legalFor(Types: allowedVectorTypes)
276 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
277 .moreElementsToNextPow2(TypeIdx: 0)
278 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
279 Mutation: LegalizeMutations::changeElementCountTo(
280 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
281 .alwaysLegal();
282
283 getActionDefinitionsBuilder(Opcode: G_INTRINSIC_W_SIDE_EFFECTS).custom();
284
285 getActionDefinitionsBuilder(Opcode: G_SHUFFLE_VECTOR)
286 .legalForCartesianProduct(Types0: allowedVectorTypes, Types1: allowedVectorTypes)
287 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
288 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 1, IsLongVecs))
289 .moreElementsToNextPow2(TypeIdx: 0)
290 .lowerIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize))
291 .moreElementsToNextPow2(TypeIdx: 1)
292 .lowerIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 1, Size: MaxVectorSize));
293
294 getActionDefinitionsBuilder(Opcode: G_EXTRACT_VECTOR_ELT)
295 .customIf(Predicate: typeOfLongVectors(TypeIdx: 1, IsLongVecs))
296 .moreElementsToNextPow2(TypeIdx: 1)
297 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 1, Size: MaxVectorSize),
298 Mutation: LegalizeMutations::changeElementCountTo(
299 TypeIdx: 1, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
300 .custom();
301
302 getActionDefinitionsBuilder(Opcode: G_INSERT_VECTOR_ELT)
303 .customIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
304 .moreElementsToNextPow2(TypeIdx: 0)
305 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
306 Mutation: LegalizeMutations::changeElementCountTo(
307 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
308 .custom();
309
310 // Illegal G_UNMERGE_VALUES instructions should be handled
311 // during the combine phase.
312 getActionDefinitionsBuilder(Opcode: G_BUILD_VECTOR)
313 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
314 .legalIf(Predicate: vectorElementCountIsLessThanOrEqualTo(TypeIdx: 0, Size: MaxVectorSize))
315 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
316 Mutation: LegalizeMutations::changeElementCountTo(
317 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)));
318
319 // When entering the legalizer, there should be no G_BITCAST instructions.
320 // They should all be calls to the `spv_bitcast` intrinsic. The call to
321 // the intrinsic will be converted to a G_BITCAST during legalization if
322 // the vectors are not legal. After using the rules to legalize a G_BITCAST,
323 // we turn it back into a call to the intrinsic with a custom rule to avoid
324 // potential machine verifier failures.
325 getActionDefinitionsBuilder(Opcode: G_BITCAST)
326 .customIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
327 .moreElementsToNextPow2(TypeIdx: 0)
328 .moreElementsToNextPow2(TypeIdx: 1)
329 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
330 Mutation: LegalizeMutations::changeElementCountTo(
331 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
332 .lowerIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 1, Size: MaxVectorSize))
333 .custom();
334
335 // If the result is still illegal, the combiner should be able to remove it.
336 getActionDefinitionsBuilder(Opcode: G_CONCAT_VECTORS)
337 .legalForCartesianProduct(Types0: allowedVectorTypes, Types1: allowedVectorTypes)
338 .legalIf(Predicate: LegalityPredicates::any(P0: typeOfLongVectors(TypeIdx: 0, IsLongVecs),
339 P1: typeOfLongVectors(TypeIdx: 1, IsLongVecs)));
340
341 getActionDefinitionsBuilder(Opcode: G_SPLAT_VECTOR)
342 .legalFor(Types: allowedVectorTypes)
343 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
344 .moreElementsToNextPow2(TypeIdx: 0)
345 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
346 Mutation: LegalizeMutations::changeElementSizeTo(TypeIdx: 0, FromTypeIdx: MaxVectorSize))
347 .alwaysLegal();
348
349 // Vector Reduction Operations
350 getActionDefinitionsBuilder(
351 Opcodes: {G_VECREDUCE_SMIN, G_VECREDUCE_SMAX, G_VECREDUCE_UMIN, G_VECREDUCE_UMAX,
352 G_VECREDUCE_ADD, G_VECREDUCE_MUL, G_VECREDUCE_FMUL, G_VECREDUCE_FMIN,
353 G_VECREDUCE_FMAX, G_VECREDUCE_FMINIMUM, G_VECREDUCE_FMAXIMUM,
354 G_VECREDUCE_OR, G_VECREDUCE_AND, G_VECREDUCE_XOR})
355 .legalFor(Types: allowedVectorTypes)
356 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
357 .scalarize(TypeIdx: 1)
358 .lower();
359
360 getActionDefinitionsBuilder(Opcodes: {G_VECREDUCE_SEQ_FADD, G_VECREDUCE_SEQ_FMUL})
361 .scalarize(TypeIdx: 2)
362 .lower();
363
364 // Illegal G_UNMERGE_VALUES instructions should be handled
365 // during the combine phase.
366 getActionDefinitionsBuilder(Opcode: G_UNMERGE_VALUES)
367 .legalIf(Predicate: LegalityPredicates::any(P0: typeOfLongVectors(TypeIdx: 0, IsLongVecs),
368 P1: typeOfLongVectors(TypeIdx: 1, IsLongVecs)))
369 .legalIf(Predicate: vectorElementCountIsLessThanOrEqualTo(TypeIdx: 1, Size: MaxVectorSize));
370
371 getActionDefinitionsBuilder(Opcodes: {G_MEMCPY, G_MEMCPY_INLINE, G_MEMMOVE})
372 .unsupportedIf(Predicate: LegalityPredicates::any(P0: typeIs(TypeIdx: 0, TypesInit: p9), P1: typeIs(TypeIdx: 1, TypesInit: p9)))
373 .legalIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allPtrs), P1: typeInSet(TypeIdx: 1, TypesInit: allPtrs)));
374
375 getActionDefinitionsBuilder(Opcodes: {G_MEMSET, G_MEMSET_INLINE})
376 .unsupportedIf(Predicate: typeIs(TypeIdx: 0, TypesInit: p9))
377 .legalIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allPtrs), P1: typeInSet(TypeIdx: 1, TypesInit: allIntScalars)));
378
379 getActionDefinitionsBuilder(Opcode: G_ADDRSPACE_CAST)
380 .legalForCartesianProduct(Types0: allPtrs, Types1: allPtrs);
381
382 // Should we be legalizing bad scalar sizes like s5 here instead
383 // of handling them in the instruction selector?
384 getActionDefinitionsBuilder(Opcodes: {G_LOAD, G_STORE})
385 .unsupportedIf(Predicate: typeIs(TypeIdx: 1, TypesInit: p9))
386 .legalForCartesianProduct(Types0: allowedVectorTypes, Types1: allPtrs)
387 .legalForCartesianProduct(Types0: allPtrs, Types1: allPtrs)
388 .legalIf(Predicate: isScalar(TypeIdx: 0))
389 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
390 .custom();
391
392 getActionDefinitionsBuilder(Opcodes: {G_SMIN, G_SMAX, G_UMIN, G_UMAX, G_ABS,
393 G_BITREVERSE, G_SADDSAT, G_UADDSAT, G_SSUBSAT,
394 G_USUBSAT, G_SCMP, G_UCMP})
395 .legalFor(Types: allIntScalars)
396 .legalFor(Types: allowedIntVectorTypes)
397 .legalIf(Predicate: ExtendedIntScalarsAndVectors)
398 // LLVM i1 maps to OpTypeBool, not OpTypeInt.
399 .scalarizeIf(Predicate: typeInSet(TypeIdx: 0, TypesInit: allBoolVectors), TypeIdx: 0)
400 .minScalar(TypeIdx: 0, Ty: s32)
401 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
402 Mutation: LegalizeMutations::changeElementCountTo(
403 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
404 .moreElementsToNextPow2(TypeIdx: 0);
405
406 getActionDefinitionsBuilder(Opcodes: {G_SSHLSAT, G_USHLSAT}).lower();
407
408 getActionDefinitionsBuilder(Opcodes: {G_FLDEXP, G_STRICT_FLDEXP})
409 .legalForCartesianProduct(Types0: allFloatScalarsAndVectors,
410 Types1: allIntScalarsAndVectors);
411
412 getActionDefinitionsBuilder(Opcodes: {G_FPTOSI, G_FPTOUI})
413 .legalForCartesianProduct(Types0: allIntScalarsAndVectors,
414 Types1: allFloatScalarsAndVectors)
415 // LLVM i1 maps to OpTypeBool, not OpTypeInt.
416 .widenScalarIf(Predicate: typeInSet(TypeIdx: 0, TypesInit: allBoolScalarsAndVectors),
417 Mutation: LegalizeMutations::changeElementTo(TypeIdx: 0, Ty: s32));
418
419 getActionDefinitionsBuilder(Opcodes: {G_FPTOSI_SAT, G_FPTOUI_SAT})
420 .legalForCartesianProduct(Types0: allIntScalarsAndVectors,
421 Types1: allFloatScalarsAndVectors);
422
423 getActionDefinitionsBuilder(Opcodes: {G_SITOFP, G_UITOFP})
424 .legalForCartesianProduct(Types0: allFloatScalarsAndVectors,
425 Types1: allScalarsAndVectors);
426
427 getActionDefinitionsBuilder(Opcode: G_CTPOP)
428 .legalForCartesianProduct(Types: allIntScalarsAndVectors)
429 .legalIf(Predicate: ExtendedScalarsAndVectorsProduct)
430 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs));
431
432 getActionDefinitionsBuilder(Opcodes: {G_TRUNC, G_ZEXT, G_SEXT, G_ANYEXT})
433 .legalForCartesianProduct(Types: allowedScalarsAndVectors)
434 .legalIf(Predicate: ExtendedScalarsAndVectorsProduct)
435 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
436 .moreElementsToNextPow2(TypeIdx: 0)
437 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
438 Mutation: LegalizeMutations::changeElementCountTo(
439 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)));
440
441 getActionDefinitionsBuilder(Opcode: G_SEXT_INREG)
442 .lowerIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
443 .moreElementsToNextPow2(TypeIdx: 0)
444 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
445 Mutation: LegalizeMutations::changeElementCountTo(
446 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
447 .lower();
448
449 getActionDefinitionsBuilder(Opcode: G_PHI)
450 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
451 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
452 Mutation: LegalizeMutations::changeElementCountTo(
453 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
454 .legalFor(Types: allPtrsScalarsAndVectors)
455 .legalIf(Predicate: ExtendedPtrsScalarsAndVectors)
456 .legalIf(Predicate: [](const LegalityQuery &Query) {
457 return Query.Types[0].isPointerVector();
458 })
459 .moreElementsToNextPow2(TypeIdx: 0);
460
461 getActionDefinitionsBuilder(Opcode: G_BITCAST).legalIf(
462 Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allPtrsScalarsAndVectors),
463 P1: typeInSet(TypeIdx: 1, TypesInit: allPtrsScalarsAndVectors)));
464
465 getActionDefinitionsBuilder(Opcodes: {G_IMPLICIT_DEF, G_FREEZE})
466 .legalFor(Types: {s1, s128})
467 .legalFor(Types: allFloatAndIntScalarsAndPtrs)
468 .legalFor(Types: allowedVectorTypes)
469 .legalIf(Predicate: [](const LegalityQuery &Query) {
470 return Query.Types[0].isPointerVector();
471 })
472 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 0, IsLongVecs))
473 .moreElementsToNextPow2(TypeIdx: 0)
474 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
475 Mutation: LegalizeMutations::changeElementCountTo(
476 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)));
477
478 getActionDefinitionsBuilder(Opcodes: {G_STACKSAVE, G_STACKRESTORE}).alwaysLegal();
479
480 getActionDefinitionsBuilder(Opcode: G_INTTOPTR)
481 .legalForCartesianProduct(Types0: allPtrs, Types1: allIntScalars)
482 .legalIf(
483 Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allPtrs), P1: typeOfExtendedScalars(TypeIdx: 1, IsExtendedInts)))
484 .legalIf(Predicate: [](const LegalityQuery &Query) {
485 const LLT DstTy = Query.Types[0];
486 const LLT SrcTy = Query.Types[1];
487 return DstTy.isPointerVector() && SrcTy.isVector() &&
488 !SrcTy.isPointer() &&
489 DstTy.getNumElements() == SrcTy.getNumElements();
490 });
491 getActionDefinitionsBuilder(Opcode: G_PTRTOINT)
492 .legalForCartesianProduct(Types0: allIntScalars, Types1: allPtrs)
493 .legalIf(
494 Predicate: all(P0: typeOfExtendedScalars(TypeIdx: 0, IsExtendedInts), P1: typeInSet(TypeIdx: 1, TypesInit: allPtrs)))
495 .legalIf(Predicate: [](const LegalityQuery &Query) {
496 const LLT DstTy = Query.Types[0];
497 const LLT SrcTy = Query.Types[1];
498 return SrcTy.isPointerVector() && DstTy.isVector() &&
499 !DstTy.isPointer() &&
500 DstTy.getNumElements() == SrcTy.getNumElements();
501 });
502 getActionDefinitionsBuilder(Opcode: G_PTR_ADD)
503 .legalForCartesianProduct(Types0: allPtrs, Types1: allIntScalars)
504 .legalIf(
505 Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allPtrs), P1: typeOfExtendedScalars(TypeIdx: 1, IsExtendedInts)));
506
507 getActionDefinitionsBuilder(Opcode: G_PTRMASK)
508 .legalForCartesianProduct(Types0: allPtrs, Types1: allIntScalars)
509 .legalIf(
510 Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allPtrs), P1: typeOfExtendedScalars(TypeIdx: 1, IsExtendedInts)))
511 .legalIf(Predicate: [](const LegalityQuery &Query) {
512 const LLT PtrTy = Query.Types[0];
513 const LLT MaskTy = Query.Types[1];
514 return PtrTy.isPointerVector() && MaskTy.isVector() &&
515 !MaskTy.isPointer() &&
516 PtrTy.getNumElements() == MaskTy.getNumElements();
517 });
518
519 // ST.canDirectlyComparePointers() for pointer args is supported in
520 // legalizeCustom().
521 getActionDefinitionsBuilder(Opcode: G_ICMP)
522 .unsupportedIf(Predicate: LegalityPredicates::any(
523 P0: all(P0: typeIs(TypeIdx: 0, TypesInit: p9), P1: typeInSet(TypeIdx: 1, TypesInit: allPtrs), args: typeIsNot(TypeIdx: 1, Type: p9)),
524 P1: all(P0: typeInSet(TypeIdx: 0, TypesInit: allPtrs), P1: typeIsNot(TypeIdx: 0, Type: p9), args: typeIs(TypeIdx: 1, TypesInit: p9))))
525 .legalIf(
526 Predicate: all(P0: typeOfLongVectors(TypeIdx: 1, IsLongVecs), P1: predNot(P: isPointerVector(TypeIdx: 1))))
527 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 1, Size: MaxVectorSize),
528 Mutation: LegalizeMutations::changeElementCountTo(
529 TypeIdx: 1, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
530 .legalIf(Predicate: [IsExtendedInts, &ST](const LegalityQuery &Query) {
531 const LLT Ty = Query.Types[1];
532 return IsExtendedInts && Ty.isValid() &&
533 !Ty.isPointerOrPointerVector() &&
534 !needsVectorLegalization(Ty, ST);
535 })
536 .customIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allBoolScalarsAndVectors),
537 P1: typeInSet(TypeIdx: 1, TypesInit: allPtrsScalarsAndVectors)))
538 .moreElementsToNextPow2(TypeIdx: 1);
539
540 getActionDefinitionsBuilder(Opcode: G_FCMP)
541 .legalIf(Predicate: typeOfLongVectors(TypeIdx: 1, IsLongVecs))
542 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 1, Size: MaxVectorSize),
543 Mutation: LegalizeMutations::changeElementCountTo(
544 TypeIdx: 1, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
545 .legalIf(Predicate: all(P0: typeInSet(TypeIdx: 0, TypesInit: allBoolScalarsAndVectors),
546 P1: typeInSet(TypeIdx: 1, TypesInit: allFloatScalarsAndVectors)))
547 .moreElementsToNextPow2(TypeIdx: 1);
548
549 getActionDefinitionsBuilder(Opcodes: {G_ATOMICRMW_OR, G_ATOMICRMW_ADD, G_ATOMICRMW_AND,
550 G_ATOMICRMW_MAX, G_ATOMICRMW_MIN,
551 G_ATOMICRMW_SUB, G_ATOMICRMW_XOR,
552 G_ATOMICRMW_UMAX, G_ATOMICRMW_UMIN})
553 .legalForCartesianProduct(Types0: allIntScalars, Types1: allPtrs);
554
555 getActionDefinitionsBuilder(
556 Opcodes: {G_ATOMICRMW_FADD, G_ATOMICRMW_FSUB, G_ATOMICRMW_FMIN, G_ATOMICRMW_FMAX})
557 .legalForCartesianProduct(Types0: allFloatScalarsAndF16Vector2AndVector4s,
558 Types1: allPtrs);
559
560 getActionDefinitionsBuilder(Opcode: G_ATOMICRMW_XCHG)
561 .legalForCartesianProduct(Types0: allFloatAndIntScalarsAndPtrs, Types1: allPtrs);
562
563 getActionDefinitionsBuilder(Opcode: G_ATOMIC_CMPXCHG_WITH_SUCCESS).lower();
564 // TODO: add proper legalization rules.
565 getActionDefinitionsBuilder(Opcode: G_ATOMIC_CMPXCHG).alwaysLegal();
566 getActionDefinitionsBuilder(Opcode: G_PREFETCH).alwaysLegal();
567
568 getActionDefinitionsBuilder(Opcodes: {G_UADDO, G_USUBO, G_UMULO, G_SMULO})
569 .alwaysLegal();
570
571 getActionDefinitionsBuilder(Opcodes: {G_SADDO, G_SSUBO}).lower();
572
573 // Lowering widens s64 to s128, which needs
574 // SPV_ALTERA_arbitrary_precision_integers. Mark s64 unsupported otherwise.
575 auto &MulFix = getActionDefinitionsBuilder(
576 Opcodes: {G_SMULFIX, G_UMULFIX, G_SMULFIXSAT, G_UMULFIXSAT});
577 if (!HasArbitraryPrecisionInts)
578 MulFix.unsupportedFor(Types: {s64});
579 MulFix.lower();
580
581 getActionDefinitionsBuilder(Opcodes: {G_TRUNC_SSAT_S, G_TRUNC_USAT_U}).lower();
582
583 getActionDefinitionsBuilder(Opcodes: {G_LROUND, G_LLROUND})
584 .legalForCartesianProduct(Types0: allIntScalarsAndVectors,
585 Types1: allFloatScalarsAndVectors);
586
587 // FP conversions.
588 getActionDefinitionsBuilder(Opcodes: {G_FPTRUNC, G_FPEXT})
589 .legalForCartesianProduct(Types: allFloatScalarsAndVectors);
590
591 // Pointer-handling.
592 getActionDefinitionsBuilder(Opcode: G_FRAME_INDEX).legalFor(Types: {p0});
593
594 getActionDefinitionsBuilder(Opcode: G_GLOBAL_VALUE).legalFor(Types: allPtrs);
595
596 // Control-flow. In some cases (e.g. constants) s1 may be promoted to s32.
597 getActionDefinitionsBuilder(Opcode: G_BR).alwaysLegal();
598 getActionDefinitionsBuilder(Opcode: G_BRCOND).legalFor(Types: {s1, s32});
599
600 getActionDefinitionsBuilder(Opcode: G_FFREXP).legalForCartesianProduct(
601 Types0: allFloatScalarsAndVectors, Types1: {s32, v2s32, v3s32, v4s32, v8s32, v16s32});
602
603 // TODO: Review the target OpenCL and GLSL Extended Instruction Set specs to
604 // tighten these requirements. Many of these math functions are only legal on
605 // specific bitwidths, so they are not selectable for
606 // allFloatScalarsAndVectors.
607 // clang-format off
608 getActionDefinitionsBuilder(Opcodes: {G_STRICT_FSQRT,
609 G_FPOW,
610 G_FEXP,
611 G_FMODF,
612 G_FSINCOS,
613 G_FEXP2,
614 G_FEXP10,
615 G_FLOG,
616 G_FLOG2,
617 G_FLOG10,
618 G_FABS,
619 G_FMINNUM,
620 G_FMAXNUM,
621 G_FCEIL,
622 G_FCOS,
623 G_FSIN,
624 G_FTAN,
625 G_FACOS,
626 G_FASIN,
627 G_FATAN,
628 G_FATAN2,
629 G_FCOSH,
630 G_FSINH,
631 G_FTANH,
632 G_FSQRT,
633 G_FFLOOR,
634 G_FRINT,
635 G_FNEARBYINT,
636 G_INTRINSIC_ROUND,
637 G_INTRINSIC_TRUNC,
638 G_INTRINSIC_ROUNDEVEN})
639 .legalFor(Types: allFloatScalars)
640 .legalFor(Types: allowedFloatVectorTypes)
641 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
642 Mutation: LegalizeMutations::changeElementCountTo(
643 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
644 .moreElementsToNextPow2(TypeIdx: 0);
645 // clang-format on
646
647 getActionDefinitionsBuilder(Opcode: G_FCOPYSIGN)
648 .legalForCartesianProduct(Types0: allFloatScalarsAndVectors,
649 Types1: allFloatScalarsAndVectors)
650 .fewerElementsIf(Predicate: vectorElementCountIsGreaterThan(TypeIdx: 0, Size: MaxVectorSize),
651 Mutation: LegalizeMutations::changeElementCountTo(
652 TypeIdx: 0, EC: ElementCount::getFixed(MinVal: MaxVectorSize)))
653 .moreElementsToNextPow2(TypeIdx: 0);
654
655 getActionDefinitionsBuilder(Opcode: G_FPOWI).legalForCartesianProduct(
656 Types0: allFloatScalarsAndVectors, Types1: allIntScalarsAndVectors);
657
658 if (ST.canUseExtInstSet(E: SPIRV::InstructionSet::OpenCL_std)) {
659 getActionDefinitionsBuilder(
660 Opcodes: {G_CTTZ, G_CTTZ_ZERO_POISON, G_CTLZ, G_CTLZ_ZERO_POISON})
661 .legalForCartesianProduct(Types0: allIntScalarsAndVectors,
662 Types1: allIntScalarsAndVectors);
663
664 // Struct return types become a single scalar, so cannot easily legalize.
665 getActionDefinitionsBuilder(Opcodes: {G_SMULH, G_UMULH}).alwaysLegal();
666 }
667
668 getActionDefinitionsBuilder(Opcode: G_IS_FPCLASS).custom();
669
670 getActionDefinitionsBuilder(Opcodes: {G_INTRINSIC, G_INTRINSIC_CONVERGENT,
671 G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS})
672 .alwaysLegal();
673 getActionDefinitionsBuilder(Opcode: G_FENCE).alwaysLegal();
674 getActionDefinitionsBuilder(Opcodes: {G_TRAP, G_DEBUGTRAP, G_UBSANTRAP}).alwaysLegal();
675
676 verify(MII: *ST.getInstrInfo());
677}
678
679static bool legalizeExtractVectorElt(LegalizerHelper &Helper,
680 MachineInstr &MI) {
681 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
682 Register DstReg = MI.getOperand(i: 0).getReg();
683 Register SrcReg = MI.getOperand(i: 1).getReg();
684 Register IdxReg = MI.getOperand(i: 2).getReg();
685
686 MIRBuilder
687 .buildIntrinsic(ID: Intrinsic::spv_extractelt, Res: ArrayRef<Register>{DstReg})
688 .addUse(RegNo: SrcReg)
689 .addUse(RegNo: IdxReg);
690 MI.eraseFromParent();
691 return true;
692}
693
694static bool legalizeInsertVectorElt(LegalizerHelper &Helper, MachineInstr &MI) {
695 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
696 Register DstReg = MI.getOperand(i: 0).getReg();
697 Register SrcReg = MI.getOperand(i: 1).getReg();
698 Register ValReg = MI.getOperand(i: 2).getReg();
699 Register IdxReg = MI.getOperand(i: 3).getReg();
700
701 MIRBuilder
702 .buildIntrinsic(ID: Intrinsic::spv_insertelt, Res: ArrayRef<Register>{DstReg})
703 .addUse(RegNo: SrcReg)
704 .addUse(RegNo: ValReg)
705 .addUse(RegNo: IdxReg);
706 MI.eraseFromParent();
707 return true;
708}
709
710static Register convertPtrToInt(Register Reg, LLT ConvTy, SPIRVTypeInst SpvType,
711 LegalizerHelper &Helper,
712 MachineRegisterInfo &MRI,
713 SPIRVGlobalRegistry *GR) {
714 Register ConvReg = MRI.createGenericVirtualRegister(Ty: ConvTy);
715 MRI.setRegClass(Reg: ConvReg, RC: GR->getRegClass(SpvType));
716 GR->assignSPIRVTypeToVReg(Type: SpvType, VReg: ConvReg, MF: Helper.MIRBuilder.getMF());
717 Helper.MIRBuilder.buildInstr(Opcode: TargetOpcode::G_PTRTOINT)
718 .addDef(RegNo: ConvReg)
719 .addUse(RegNo: Reg);
720 return ConvReg;
721}
722
723static bool legalizeLoad(LegalizerHelper &Helper, MachineInstr &MI,
724 SPIRVGlobalRegistry *GR) {
725 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
726 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
727 Register DstReg = MI.getOperand(i: 0).getReg();
728 Register PtrReg = MI.getOperand(i: 1).getReg();
729 LLT DstTy = MRI.getType(Reg: DstReg);
730
731 if (!DstTy.isVector())
732 return true;
733
734 const SPIRVSubtarget &ST = MI.getMF()->getSubtarget<SPIRVSubtarget>();
735 if (!needsVectorLegalization(Ty: DstTy, ST))
736 return true;
737
738 SmallVector<Register, 8> SplitRegs;
739 LLT EltTy = DstTy.getElementType();
740 unsigned NumElts = DstTy.getNumElements();
741
742 LLT PtrTy = MRI.getType(Reg: PtrReg);
743 auto Zero = MIRBuilder.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: 0);
744
745 for (unsigned i = 0; i < NumElts; ++i) {
746 auto Idx = MIRBuilder.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: i);
747 Register EltPtr = MRI.createGenericVirtualRegister(Ty: PtrTy);
748
749 MIRBuilder.buildIntrinsic(ID: Intrinsic::spv_gep, Res: ArrayRef<Register>{EltPtr})
750 .addImm(Val: 1) // InBounds
751 .addUse(RegNo: PtrReg)
752 .addUse(RegNo: Zero.getReg(Idx: 0))
753 .addUse(RegNo: Idx.getReg(Idx: 0));
754
755 MachinePointerInfo EltPtrInfo;
756 Align EltAlign = Align(1);
757 if (!MI.memoperands_empty()) {
758 MachineMemOperand *MMO = *MI.memoperands_begin();
759 EltPtrInfo =
760 MMO->getPointerInfo().getWithOffset(O: i * EltTy.getSizeInBytes());
761 EltAlign = commonAlignment(A: MMO->getAlign(), Offset: i * EltTy.getSizeInBytes());
762 }
763
764 Register EltReg = MRI.createGenericVirtualRegister(Ty: EltTy);
765 MIRBuilder.buildLoad(Res: EltReg, Addr: EltPtr, PtrInfo: EltPtrInfo, Alignment: EltAlign);
766 SplitRegs.push_back(Elt: EltReg);
767 }
768
769 MIRBuilder.buildBuildVector(Res: DstReg, Ops: SplitRegs);
770 MI.eraseFromParent();
771 return true;
772}
773
774static bool legalizeStore(LegalizerHelper &Helper, MachineInstr &MI,
775 SPIRVGlobalRegistry *GR) {
776 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
777 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
778 Register ValReg = MI.getOperand(i: 0).getReg();
779 Register PtrReg = MI.getOperand(i: 1).getReg();
780 LLT ValTy = MRI.getType(Reg: ValReg);
781
782 assert(ValTy.isVector() && "Expected vector store");
783
784 SmallVector<Register, 8> SplitRegs;
785 LLT EltTy = ValTy.getElementType();
786 unsigned NumElts = ValTy.getNumElements();
787
788 for (unsigned i = 0; i < NumElts; ++i)
789 SplitRegs.push_back(Elt: MRI.createGenericVirtualRegister(Ty: EltTy));
790
791 MIRBuilder.buildUnmerge(Res: SplitRegs, Op: ValReg);
792
793 LLT PtrTy = MRI.getType(Reg: PtrReg);
794 auto Zero = MIRBuilder.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: 0);
795
796 for (unsigned i = 0; i < NumElts; ++i) {
797 auto Idx = MIRBuilder.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: i);
798 Register EltPtr = MRI.createGenericVirtualRegister(Ty: PtrTy);
799
800 MIRBuilder.buildIntrinsic(ID: Intrinsic::spv_gep, Res: ArrayRef<Register>{EltPtr})
801 .addImm(Val: 1) // InBounds
802 .addUse(RegNo: PtrReg)
803 .addUse(RegNo: Zero.getReg(Idx: 0))
804 .addUse(RegNo: Idx.getReg(Idx: 0));
805
806 MachinePointerInfo EltPtrInfo;
807 Align EltAlign = Align(1);
808 if (!MI.memoperands_empty()) {
809 MachineMemOperand *MMO = *MI.memoperands_begin();
810 EltPtrInfo =
811 MMO->getPointerInfo().getWithOffset(O: i * EltTy.getSizeInBytes());
812 EltAlign = commonAlignment(A: MMO->getAlign(), Offset: i * EltTy.getSizeInBytes());
813 }
814
815 MIRBuilder.buildStore(Val: SplitRegs[i], Addr: EltPtr, PtrInfo: EltPtrInfo, Alignment: EltAlign);
816 }
817
818 MI.eraseFromParent();
819 return true;
820}
821
822bool SPIRVLegalizerInfo::legalizeCustom(
823 LegalizerHelper &Helper, MachineInstr &MI,
824 LostDebugLocObserver &LocObserver) const {
825 MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
826 switch (MI.getOpcode()) {
827 default:
828 // TODO: implement legalization for other opcodes.
829 return true;
830 case TargetOpcode::G_BITCAST:
831 return legalizeBitcast(Helper, MI);
832 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
833 return legalizeExtractVectorElt(Helper, MI);
834 case TargetOpcode::G_INSERT_VECTOR_ELT:
835 return legalizeInsertVectorElt(Helper, MI);
836 case TargetOpcode::G_INTRINSIC:
837 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
838 return legalizeIntrinsic(Helper, MI);
839 case TargetOpcode::G_IS_FPCLASS:
840 return legalizeIsFPClass(Helper, MI, LocObserver);
841 case TargetOpcode::G_ICMP: {
842 auto &Op0 = MI.getOperand(i: 2);
843 auto &Op1 = MI.getOperand(i: 3);
844 Register Reg0 = Op0.getReg();
845 Register Reg1 = Op1.getReg();
846 CmpInst::Predicate Cond =
847 static_cast<CmpInst::Predicate>(MI.getOperand(i: 1).getPredicate());
848 if ((!ST->canDirectlyComparePointers() ||
849 (Cond != CmpInst::ICMP_EQ && Cond != CmpInst::ICMP_NE)) &&
850 MRI.getType(Reg: Reg0).isPointer() && MRI.getType(Reg: Reg1).isPointer()) {
851 LLT ConvT = LLT::scalar(SizeInBits: ST->getPointerSize());
852 Type *LLVMTy = IntegerType::get(C&: MI.getMF()->getFunction().getContext(),
853 NumBits: ST->getPointerSize());
854 SPIRVTypeInst SpirvTy = GR->getOrCreateSPIRVType(
855 Type: LLVMTy, MIRBuilder&: Helper.MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
856 Op0.setReg(convertPtrToInt(Reg: Reg0, ConvTy: ConvT, SpvType: SpirvTy, Helper, MRI, GR));
857 Op1.setReg(convertPtrToInt(Reg: Reg1, ConvTy: ConvT, SpvType: SpirvTy, Helper, MRI, GR));
858 }
859 return true;
860 }
861 case TargetOpcode::G_LOAD:
862 return legalizeLoad(Helper, MI, GR);
863 case TargetOpcode::G_STORE:
864 return legalizeStore(Helper, MI, GR);
865 }
866}
867
868static MachineInstrBuilder
869createStackTemporaryForVector(LegalizerHelper &Helper, SPIRVGlobalRegistry *GR,
870 Register SrcReg, LLT SrcTy,
871 MachinePointerInfo &PtrInfo, Align &VecAlign) {
872 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
873 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
874
875 VecAlign = Helper.getStackTemporaryAlignment(Type: SrcTy);
876 auto StackTemp = Helper.createStackTemporary(
877 Bytes: TypeSize::getFixed(ExactSize: SrcTy.getSizeInBytes()), Alignment: VecAlign, PtrInfo);
878
879 // Set the type of StackTemp to a pointer to an array of the element type.
880 SPIRVTypeInst SpvSrcTy = GR->getSPIRVTypeForVReg(VReg: SrcReg);
881 SPIRVTypeInst EltSpvTy = GR->getScalarOrVectorComponentType(Type: SpvSrcTy);
882 const Type *LLVMEltTy = GR->getTypeForSPIRVType(Ty: EltSpvTy);
883 const Type *LLVMArrTy =
884 ArrayType::get(ElementType: const_cast<Type *>(LLVMEltTy), NumElements: SrcTy.getNumElements());
885 SPIRVTypeInst ArrSpvTy = GR->getOrCreateSPIRVType(
886 Type: LLVMArrTy, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite, EmitIR: true);
887 SPIRVTypeInst PtrToArrSpvTy = GR->getOrCreateSPIRVPointerType(
888 BaseType: ArrSpvTy, MIRBuilder, SC: SPIRV::StorageClass::Function);
889
890 Register StackReg = StackTemp.getReg(Idx: 0);
891 MRI.setRegClass(Reg: StackReg, RC: GR->getRegClass(SpvType: PtrToArrSpvTy));
892 GR->assignSPIRVTypeToVReg(Type: PtrToArrSpvTy, VReg: StackReg, MF: MIRBuilder.getMF());
893
894 return StackTemp;
895}
896
897static bool legalizeSpvBitcast(LegalizerHelper &Helper, MachineInstr &MI,
898 SPIRVGlobalRegistry *GR) {
899 LLVM_DEBUG(dbgs() << "Found a bitcast instruction\n");
900 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
901 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
902 const SPIRVSubtarget &ST = MI.getMF()->getSubtarget<SPIRVSubtarget>();
903
904 Register DstReg = MI.getOperand(i: 0).getReg();
905 Register SrcReg = MI.getOperand(i: 2).getReg();
906 LLT DstTy = MRI.getType(Reg: DstReg);
907 LLT SrcTy = MRI.getType(Reg: SrcReg);
908
909 // If an spv_bitcast needs to be legalized, we convert it to G_BITCAST to
910 // allow using the generic legalization rules.
911 if (needsVectorLegalization(Ty: DstTy, ST) ||
912 needsVectorLegalization(Ty: SrcTy, ST)) {
913 LLVM_DEBUG(dbgs() << "Replacing with a G_BITCAST\n");
914 MIRBuilder.buildBitcast(Dst: DstReg, Src: SrcReg);
915 MI.eraseFromParent();
916 }
917 return true;
918}
919
920static bool legalizeSpvInsertElt(LegalizerHelper &Helper, MachineInstr &MI,
921 SPIRVGlobalRegistry *GR) {
922 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
923 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
924 const SPIRVSubtarget &ST = MI.getMF()->getSubtarget<SPIRVSubtarget>();
925
926 Register DstReg = MI.getOperand(i: 0).getReg();
927 LLT DstTy = MRI.getType(Reg: DstReg);
928
929 if (needsVectorLegalization(Ty: DstTy, ST)) {
930 Register SrcReg = MI.getOperand(i: 2).getReg();
931 Register ValReg = MI.getOperand(i: 3).getReg();
932 LLT SrcTy = MRI.getType(Reg: SrcReg);
933 MachineOperand &IdxOperand = MI.getOperand(i: 4);
934
935 if (getImm(MO: IdxOperand, MRI: &MRI)) {
936 uint64_t IdxVal = foldImm(MO: IdxOperand, MRI: &MRI);
937 if (IdxVal < SrcTy.getNumElements()) {
938 SmallVector<Register, 8> Regs;
939 SPIRVTypeInst ElementType =
940 GR->getScalarOrVectorComponentType(Type: GR->getSPIRVTypeForVReg(VReg: DstReg));
941 LLT ElementLLTTy = GR->getRegType(SpvType: ElementType);
942 for (unsigned I = 0, E = SrcTy.getNumElements(); I < E; ++I) {
943 Register Reg = MRI.createGenericVirtualRegister(Ty: ElementLLTTy);
944 MRI.setRegClass(Reg, RC: GR->getRegClass(SpvType: ElementType));
945 GR->assignSPIRVTypeToVReg(Type: ElementType, VReg: Reg, MF: *MI.getMF());
946 Regs.push_back(Elt: Reg);
947 }
948 MIRBuilder.buildUnmerge(Res: Regs, Op: SrcReg);
949 Regs[IdxVal] = ValReg;
950 MIRBuilder.buildBuildVector(Res: DstReg, Ops: Regs);
951 MI.eraseFromParent();
952 return true;
953 }
954 }
955
956 LLT EltTy = SrcTy.getElementType();
957 Align VecAlign;
958 MachinePointerInfo PtrInfo;
959 auto StackTemp = createStackTemporaryForVector(Helper, GR, SrcReg, SrcTy,
960 PtrInfo, VecAlign);
961
962 MIRBuilder.buildStore(Val: SrcReg, Addr: StackTemp, PtrInfo, Alignment: VecAlign);
963
964 Register IdxReg = IdxOperand.getReg();
965 LLT PtrTy = MRI.getType(Reg: StackTemp.getReg(Idx: 0));
966 Register EltPtr = MRI.createGenericVirtualRegister(Ty: PtrTy);
967 auto Zero = MIRBuilder.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: 0);
968
969 MIRBuilder.buildIntrinsic(ID: Intrinsic::spv_gep, Res: ArrayRef<Register>{EltPtr})
970 .addImm(Val: 1) // InBounds
971 .addUse(RegNo: StackTemp.getReg(Idx: 0))
972 .addUse(RegNo: Zero.getReg(Idx: 0))
973 .addUse(RegNo: IdxReg);
974
975 MachinePointerInfo EltPtrInfo = MachinePointerInfo(PtrTy.getAddressSpace());
976 Align EltAlign = Helper.getStackTemporaryAlignment(Type: EltTy);
977 MIRBuilder.buildStore(Val: ValReg, Addr: EltPtr, PtrInfo: EltPtrInfo, Alignment: EltAlign);
978
979 MIRBuilder.buildLoad(Res: DstReg, Addr: StackTemp, PtrInfo, Alignment: VecAlign);
980 MI.eraseFromParent();
981 return true;
982 }
983 return true;
984}
985
986static bool legalizeSpvExtractElt(LegalizerHelper &Helper, MachineInstr &MI,
987 SPIRVGlobalRegistry *GR) {
988 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
989 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
990 const SPIRVSubtarget &ST = MI.getMF()->getSubtarget<SPIRVSubtarget>();
991
992 Register SrcReg = MI.getOperand(i: 2).getReg();
993 LLT SrcTy = MRI.getType(Reg: SrcReg);
994
995 if (needsVectorLegalization(Ty: SrcTy, ST)) {
996 Register DstReg = MI.getOperand(i: 0).getReg();
997 MachineOperand &IdxOperand = MI.getOperand(i: 3);
998
999 if (getImm(MO: IdxOperand, MRI: &MRI)) {
1000 uint64_t IdxVal = foldImm(MO: IdxOperand, MRI: &MRI);
1001 if (IdxVal < SrcTy.getNumElements()) {
1002 LLT DstTy = MRI.getType(Reg: DstReg);
1003 SmallVector<Register, 8> Regs;
1004 SPIRVTypeInst DstSpvTy = GR->getSPIRVTypeForVReg(VReg: DstReg);
1005 for (unsigned I = 0, E = SrcTy.getNumElements(); I < E; ++I) {
1006 if (I == IdxVal) {
1007 Regs.push_back(Elt: DstReg);
1008 } else {
1009 Register Reg = MRI.createGenericVirtualRegister(Ty: DstTy);
1010 MRI.setRegClass(Reg, RC: GR->getRegClass(SpvType: DstSpvTy));
1011 GR->assignSPIRVTypeToVReg(Type: DstSpvTy, VReg: Reg, MF: *MI.getMF());
1012 Regs.push_back(Elt: Reg);
1013 }
1014 }
1015 MIRBuilder.buildUnmerge(Res: Regs, Op: SrcReg);
1016 MI.eraseFromParent();
1017 return true;
1018 }
1019 }
1020
1021 LLT EltTy = SrcTy.getElementType();
1022 Align VecAlign;
1023 MachinePointerInfo PtrInfo;
1024 auto StackTemp = createStackTemporaryForVector(Helper, GR, SrcReg, SrcTy,
1025 PtrInfo, VecAlign);
1026
1027 MIRBuilder.buildStore(Val: SrcReg, Addr: StackTemp, PtrInfo, Alignment: VecAlign);
1028
1029 Register IdxReg = IdxOperand.getReg();
1030 LLT PtrTy = MRI.getType(Reg: StackTemp.getReg(Idx: 0));
1031 Register EltPtr = MRI.createGenericVirtualRegister(Ty: PtrTy);
1032 auto Zero = MIRBuilder.buildConstant(Res: LLT::scalar(SizeInBits: 32), Val: 0);
1033
1034 MIRBuilder.buildIntrinsic(ID: Intrinsic::spv_gep, Res: ArrayRef<Register>{EltPtr})
1035 .addImm(Val: 1) // InBounds
1036 .addUse(RegNo: StackTemp.getReg(Idx: 0))
1037 .addUse(RegNo: Zero.getReg(Idx: 0))
1038 .addUse(RegNo: IdxReg);
1039
1040 MachinePointerInfo EltPtrInfo = MachinePointerInfo(PtrTy.getAddressSpace());
1041 Align EltAlign = Helper.getStackTemporaryAlignment(Type: EltTy);
1042 MIRBuilder.buildLoad(Res: DstReg, Addr: EltPtr, PtrInfo: EltPtrInfo, Alignment: EltAlign);
1043
1044 MI.eraseFromParent();
1045 return true;
1046 }
1047 return true;
1048}
1049
1050static bool legalizeSpvConstComposite(LegalizerHelper &Helper, MachineInstr &MI,
1051 SPIRVGlobalRegistry *GR) {
1052 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1053 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
1054 const SPIRVSubtarget &ST = MI.getMF()->getSubtarget<SPIRVSubtarget>();
1055
1056 Register DstReg = MI.getOperand(i: 0).getReg();
1057 LLT DstTy = MRI.getType(Reg: DstReg);
1058
1059 if (!needsVectorLegalization(Ty: DstTy, ST))
1060 return true;
1061
1062 SmallVector<Register, 8> SrcRegs;
1063 if (MI.getNumOperands() == 2) {
1064 // The "null" case: no values are attached.
1065 LLT EltTy = DstTy.getElementType();
1066 auto Zero = MIRBuilder.buildConstant(Res: EltTy, Val: 0);
1067 SPIRVTypeInst SpvDstTy = GR->getSPIRVTypeForVReg(VReg: DstReg);
1068 SPIRVTypeInst SpvEltTy = GR->getScalarOrVectorComponentType(Type: SpvDstTy);
1069 GR->assignSPIRVTypeToVReg(Type: SpvEltTy, VReg: Zero.getReg(Idx: 0), MF: MIRBuilder.getMF());
1070 for (unsigned i = 0; i < DstTy.getNumElements(); ++i)
1071 SrcRegs.push_back(Elt: Zero.getReg(Idx: 0));
1072 } else {
1073 for (unsigned i = 2; i < MI.getNumOperands(); ++i) {
1074 SrcRegs.push_back(Elt: MI.getOperand(i).getReg());
1075 }
1076 }
1077 MIRBuilder.buildBuildVector(Res: DstReg, Ops: SrcRegs);
1078 MI.eraseFromParent();
1079 return true;
1080}
1081
1082static SmallVector<Register, 16> unmergeToScalars(Register Reg,
1083 MachineIRBuilder &MIRBuilder,
1084 SPIRVGlobalRegistry *GR) {
1085 LLT Ty = MIRBuilder.getMRI()->getType(Reg);
1086 if (!Ty.isVector())
1087 return {Reg};
1088 SPIRVTypeInst EltSpvTy =
1089 GR->getScalarOrVectorComponentType(Type: GR->getSPIRVTypeForVReg(VReg: Reg));
1090 unsigned NumElts = Ty.getNumElements();
1091 SmallVector<Register, 16> Elts;
1092 for (unsigned I = 0; I < NumElts; ++I)
1093 Elts.push_back(Elt: createVirtualRegister(SpvType: EltSpvTy, GR, MIRBuilder));
1094 MIRBuilder.buildUnmerge(Res: Elts, Op: Reg);
1095 return Elts;
1096}
1097
1098static Register buildVectorPart(ArrayRef<Register> Elts,
1099 MachineIRBuilder &MIRBuilder,
1100 SPIRVGlobalRegistry *GR) {
1101 if (Elts.size() == 1)
1102 return Elts[0];
1103 SPIRVTypeInst PartSpvTy =
1104 GR->getOrCreateSPIRVVectorType(BaseType: GR->getSPIRVTypeForVReg(VReg: Elts[0]),
1105 NumElements: Elts.size(), MIRBuilder, /*EmitIR=*/true);
1106 Register Part = createVirtualRegister(SpvType: PartSpvTy, GR, MIRBuilder);
1107 MIRBuilder.buildBuildVector(Res: Part, Ops: Elts);
1108 return Part;
1109}
1110
1111// Split an elementwise intrinsic with an illegal vector width into intrinsics
1112// on legal vector widths.
1113static bool legalizeElementwiseIntrinsic(LegalizerHelper &Helper,
1114 GIntrinsic &MI,
1115 SPIRVGlobalRegistry *GR) {
1116 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1117 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
1118 const SPIRVSubtarget &ST = MI.getMF()->getSubtarget<SPIRVSubtarget>();
1119
1120 if (!Intrinsic::isTriviallyScalarizable(id: MI.getIntrinsicID()))
1121 return true;
1122 Register DstReg = MI.getReg(Idx: 0);
1123 LLT DstTy = MRI.getType(Reg: DstReg);
1124 if (!needsVectorLegalization(Ty: DstTy, ST))
1125 return true;
1126
1127 unsigned NumElts = DstTy.getNumElements();
1128 unsigned MaxVectorSize = ST.isShader() ? 4 : 16;
1129 unsigned PartSize = NumElts > MaxVectorSize ? MaxVectorSize : 4;
1130
1131 SmallDenseMap<Register, SmallVector<Register, 16>, 4> OpElts;
1132 for (const MachineOperand &MO : drop_begin(RangeOrContainer: MI.explicit_uses())) {
1133 if (!MO.isReg() || !MRI.getType(Reg: MO.getReg()).isVector())
1134 continue;
1135 auto [It, Inserted] = OpElts.try_emplace(Key: MO.getReg());
1136 if (Inserted)
1137 It->second = unmergeToScalars(Reg: MO.getReg(), MIRBuilder, GR);
1138 }
1139
1140 SPIRVTypeInst DstEltSpvTy =
1141 GR->getScalarOrVectorComponentType(Type: GR->getSPIRVTypeForVReg(VReg: DstReg));
1142 SmallVector<Register, 16> DstElts;
1143 for (unsigned Offset = 0; Offset < NumElts; Offset += PartSize) {
1144 unsigned Size = std::min(a: PartSize, b: NumElts - Offset);
1145 SPIRVTypeInst PartSpvTy =
1146 Size == 1 ? DstEltSpvTy
1147 : GR->getOrCreateSPIRVVectorType(BaseType: DstEltSpvTy, NumElements: Size,
1148 MIRBuilder, /*EmitIR=*/true);
1149 SmallDenseMap<Register, Register, 4> PartRegs;
1150 SmallVector<MachineOperand> PartOps;
1151 for (const MachineOperand &MO : drop_begin(RangeOrContainer: MI.explicit_uses())) {
1152 auto EltsIt = MO.isReg() ? OpElts.find(Val: MO.getReg()) : OpElts.end();
1153 if (EltsIt == OpElts.end()) {
1154 PartOps.push_back(Elt: MO);
1155 continue;
1156 }
1157 auto [It, Inserted] = PartRegs.try_emplace(Key: MO.getReg());
1158 if (Inserted)
1159 It->second = buildVectorPart(
1160 Elts: ArrayRef(EltsIt->second).slice(N: Offset, M: Size), MIRBuilder, GR);
1161 PartOps.push_back(Elt: MachineOperand::CreateReg(Reg: It->second, /*isDef=*/false));
1162 }
1163 Register PartDst = createVirtualRegister(SpvType: PartSpvTy, GR, MIRBuilder);
1164 auto Part = MIRBuilder.buildIntrinsic(
1165 ID: MI.getIntrinsicID(), Res: ArrayRef<Register>{PartDst}, HasSideEffects: MI.hasSideEffects(),
1166 isConvergent: MI.isConvergent());
1167 for (const MachineOperand &MO : PartOps)
1168 Part.add(MO);
1169 Part->setFlags(MI.getFlags());
1170 append_range(C&: DstElts, R: unmergeToScalars(Reg: PartDst, MIRBuilder, GR));
1171 }
1172
1173 MIRBuilder.buildBuildVector(Res: DstReg, Ops: DstElts);
1174 MI.eraseFromParent();
1175 return true;
1176}
1177
1178bool SPIRVLegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
1179 MachineInstr &MI) const {
1180 LLVM_DEBUG(dbgs() << "legalizeIntrinsic: " << MI);
1181 auto IntrinsicID = cast<GIntrinsic>(Val&: MI).getIntrinsicID();
1182 switch (IntrinsicID) {
1183 case Intrinsic::spv_bitcast:
1184 return legalizeSpvBitcast(Helper, MI, GR);
1185 case Intrinsic::spv_insertelt:
1186 return legalizeSpvInsertElt(Helper, MI, GR);
1187 case Intrinsic::spv_extractelt:
1188 return legalizeSpvExtractElt(Helper, MI, GR);
1189 case Intrinsic::spv_const_composite:
1190 return legalizeSpvConstComposite(Helper, MI, GR);
1191 }
1192 return legalizeElementwiseIntrinsic(Helper, MI&: cast<GIntrinsic>(Val&: MI), GR);
1193}
1194
1195bool SPIRVLegalizerInfo::legalizeBitcast(LegalizerHelper &Helper,
1196 MachineInstr &MI) const {
1197 // Once the G_BITCAST is using vectors that are allowed, we turn it back into
1198 // an spv_bitcast to avoid verifier problems when the register types are the
1199 // same for the source and the result. Note that the SPIR-V types associated
1200 // with the bitcast can be different even if the register types are the same.
1201 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1202 Register DstReg = MI.getOperand(i: 0).getReg();
1203 Register SrcReg = MI.getOperand(i: 1).getReg();
1204 SmallVector<Register, 1> DstRegs = {DstReg};
1205 MIRBuilder.buildIntrinsic(ID: Intrinsic::spv_bitcast, Res: DstRegs).addUse(RegNo: SrcReg);
1206 MI.eraseFromParent();
1207 return true;
1208}
1209
1210// Note this code was copied from LegalizerHelper::lowerISFPCLASS and adjusted
1211// to ensure that all instructions created during the lowering have SPIR-V types
1212// assigned to them.
1213bool SPIRVLegalizerInfo::legalizeIsFPClass(
1214 LegalizerHelper &Helper, MachineInstr &MI,
1215 LostDebugLocObserver &LocObserver) const {
1216 auto [DstReg, DstTy, SrcReg, SrcTy] = MI.getFirst2RegLLTs();
1217 FPClassTest Mask = static_cast<FPClassTest>(MI.getOperand(i: 2).getImm());
1218
1219 auto &MIRBuilder = Helper.MIRBuilder;
1220 auto &MF = MIRBuilder.getMF();
1221 MachineRegisterInfo &MRI = MF.getRegInfo();
1222
1223 Type *LLVMDstTy =
1224 IntegerType::get(C&: MIRBuilder.getContext(), NumBits: DstTy.getScalarSizeInBits());
1225 if (DstTy.isVector())
1226 LLVMDstTy = VectorType::get(ElementType: LLVMDstTy, EC: DstTy.getElementCount());
1227 SPIRVTypeInst SPIRVDstTy = GR->getOrCreateSPIRVType(
1228 Type: LLVMDstTy, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite,
1229 /*EmitIR*/ true);
1230
1231 unsigned BitSize = SrcTy.getScalarSizeInBits();
1232 const fltSemantics &Semantics = getFltSemanticForLLT(Ty: SrcTy.getScalarType());
1233
1234 LLT IntTy = LLT::scalar(SizeInBits: BitSize);
1235 Type *LLVMIntTy = IntegerType::get(C&: MIRBuilder.getContext(), NumBits: BitSize);
1236 if (SrcTy.isVector()) {
1237 IntTy = LLT::vector(EC: SrcTy.getElementCount(), ScalarTy: IntTy);
1238 LLVMIntTy = VectorType::get(ElementType: LLVMIntTy, EC: SrcTy.getElementCount());
1239 }
1240 SPIRVTypeInst SPIRVIntTy = GR->getOrCreateSPIRVType(
1241 Type: LLVMIntTy, MIRBuilder, AQ: SPIRV::AccessQualifier::ReadWrite,
1242 /*EmitIR*/ true);
1243
1244 // Clang doesn't support capture of structured bindings:
1245 LLT DstTyCopy = DstTy;
1246 const auto assignSPIRVTy = [&](MachineInstrBuilder &&MI) {
1247 // Assign this MI's (assumed only) destination to one of the two types we
1248 // expect: either the G_IS_FPCLASS's destination type, or the integer type
1249 // bitcast from the source type.
1250 LLT MITy = MRI.getType(Reg: MI.getReg(Idx: 0));
1251 assert((MITy == IntTy || MITy == DstTyCopy) &&
1252 "Unexpected LLT type while lowering G_IS_FPCLASS");
1253 SPIRVTypeInst SPVTy = MITy == IntTy ? SPIRVIntTy : SPIRVDstTy;
1254 GR->assignSPIRVTypeToVReg(Type: SPVTy, VReg: MI.getReg(Idx: 0), MF);
1255 return MI;
1256 };
1257
1258 // Helper to build and assign a constant in one go
1259 const auto buildSPIRVConstant = [&](LLT Ty, auto &&C) -> MachineInstrBuilder {
1260 if (!Ty.isFixedVector())
1261 return assignSPIRVTy(MIRBuilder.buildConstant(Ty, C));
1262 auto ScalarC = MIRBuilder.buildConstant(Ty.getScalarType(), C);
1263 assert((Ty == IntTy || Ty == DstTyCopy) &&
1264 "Unexpected LLT type while lowering constant for G_IS_FPCLASS");
1265 SPIRVTypeInst VecEltTy = GR->getOrCreateSPIRVType(
1266 Type: (Ty == IntTy ? LLVMIntTy : LLVMDstTy)->getScalarType(), MIRBuilder,
1267 AQ: SPIRV::AccessQualifier::ReadWrite,
1268 /*EmitIR*/ true);
1269 GR->assignSPIRVTypeToVReg(Type: VecEltTy, VReg: ScalarC.getReg(0), MF);
1270 return assignSPIRVTy(MIRBuilder.buildSplatBuildVector(Res: Ty, Src: ScalarC));
1271 };
1272
1273 if (Mask == fcNone) {
1274 MIRBuilder.buildCopy(Res: DstReg, Op: buildSPIRVConstant(DstTy, 0));
1275 MI.eraseFromParent();
1276 return true;
1277 }
1278 if (Mask == fcAllFlags) {
1279 MIRBuilder.buildCopy(Res: DstReg, Op: buildSPIRVConstant(DstTy, 1));
1280 MI.eraseFromParent();
1281 return true;
1282 }
1283
1284 // Note that rather than creating a COPY here (between a floating-point and
1285 // integer type of the same size) we create a SPIR-V bitcast immediately. We
1286 // can't create a G_BITCAST because the LLTs are the same, and we can't seem
1287 // to correctly lower COPYs to SPIR-V bitcasts at this moment.
1288 Register ResVReg = MRI.createGenericVirtualRegister(Ty: IntTy);
1289 MRI.setRegClass(Reg: ResVReg, RC: GR->getRegClass(SpvType: SPIRVIntTy));
1290 GR->assignSPIRVTypeToVReg(Type: SPIRVIntTy, VReg: ResVReg, MF: Helper.MIRBuilder.getMF());
1291 auto AsInt = MIRBuilder.buildInstr(Opcode: SPIRV::OpBitcast)
1292 .addDef(RegNo: ResVReg)
1293 .addUse(RegNo: GR->getSPIRVTypeID(SpirvType: SPIRVIntTy))
1294 .addUse(RegNo: SrcReg);
1295 AsInt = assignSPIRVTy(std::move(AsInt));
1296
1297 // Various masks.
1298 APInt SignBit = APInt::getSignMask(BitWidth: BitSize);
1299 APInt ValueMask = APInt::getSignedMaxValue(numBits: BitSize); // All bits but sign.
1300 APInt Inf = APFloat::getInf(Sem: Semantics).bitcastToAPInt(); // Exp and int bit.
1301 APInt ExpMask = Inf;
1302 APInt AllOneMantissa = APFloat::getLargest(Sem: Semantics).bitcastToAPInt() & ~Inf;
1303 APInt QNaNBitMask =
1304 APInt::getOneBitSet(numBits: BitSize, BitNo: AllOneMantissa.getActiveBits() - 1);
1305 APInt InversionMask = APInt::getAllOnes(numBits: DstTy.getScalarSizeInBits());
1306
1307 auto SignBitC = buildSPIRVConstant(IntTy, SignBit);
1308 auto ValueMaskC = buildSPIRVConstant(IntTy, ValueMask);
1309 auto InfC = buildSPIRVConstant(IntTy, Inf);
1310 auto ExpMaskC = buildSPIRVConstant(IntTy, ExpMask);
1311 auto ZeroC = buildSPIRVConstant(IntTy, 0);
1312
1313 auto Abs = assignSPIRVTy(MIRBuilder.buildAnd(Dst: IntTy, Src0: AsInt, Src1: ValueMaskC));
1314 auto Sign = assignSPIRVTy(
1315 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_NE, Res: DstTy, Op0: AsInt, Op1: Abs));
1316
1317 MachineInstrBuilder Res;
1318
1319 const auto appendToRes = [&](MachineInstrBuilder &&ToAppend) {
1320 auto Appended = assignSPIRVTy(std::move(ToAppend));
1321 Res = Res.getInstr()
1322 ? assignSPIRVTy(MIRBuilder.buildOr(Dst: DstTyCopy, Src0: Res, Src1: Appended))
1323 : Appended;
1324 };
1325
1326 // Tests that involve more than one class should be processed first.
1327 if ((Mask & fcFinite) == fcFinite) {
1328 // finite(V) ==> abs(V) u< exp_mask
1329 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: Abs,
1330 Op1: ExpMaskC));
1331 Mask &= ~fcFinite;
1332 } else if ((Mask & fcFinite) == fcPosFinite) {
1333 // finite(V) && V > 0 ==> V u< exp_mask
1334 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: AsInt,
1335 Op1: ExpMaskC));
1336 Mask &= ~fcPosFinite;
1337 } else if ((Mask & fcFinite) == fcNegFinite) {
1338 // finite(V) && V < 0 ==> abs(V) u< exp_mask && signbit == 1
1339 auto Cmp = assignSPIRVTy(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT,
1340 Res: DstTy, Op0: Abs, Op1: ExpMaskC));
1341 appendToRes(MIRBuilder.buildAnd(Dst: DstTy, Src0: Cmp, Src1: Sign));
1342 Mask &= ~fcNegFinite;
1343 }
1344
1345 if (FPClassTest PartialCheck = Mask & (fcZero | fcSubnormal)) {
1346 // fcZero | fcSubnormal => test all exponent bits are 0
1347 // TODO: Handle sign bit specific cases
1348 // TODO: Handle inverted case
1349 if (PartialCheck == (fcZero | fcSubnormal)) {
1350 auto ExpBits = assignSPIRVTy(MIRBuilder.buildAnd(Dst: IntTy, Src0: AsInt, Src1: ExpMaskC));
1351 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
1352 Op0: ExpBits, Op1: ZeroC));
1353 Mask &= ~PartialCheck;
1354 }
1355 }
1356
1357 // Check for individual classes.
1358 if (FPClassTest PartialCheck = Mask & fcZero) {
1359 if (PartialCheck == fcPosZero)
1360 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
1361 Op0: AsInt, Op1: ZeroC));
1362 else if (PartialCheck == fcZero)
1363 appendToRes(
1364 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy, Op0: Abs, Op1: ZeroC));
1365 else // fcNegZero
1366 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
1367 Op0: AsInt, Op1: SignBitC));
1368 }
1369
1370 if (FPClassTest PartialCheck = Mask & fcSubnormal) {
1371 // issubnormal(V) ==> unsigned(abs(V) - 1) u< (all mantissa bits set)
1372 // issubnormal(V) && V>0 ==> unsigned(V - 1) u< (all mantissa bits set)
1373 auto V = (PartialCheck == fcPosSubnormal) ? AsInt : Abs;
1374 auto OneC = buildSPIRVConstant(IntTy, 1);
1375 auto VMinusOne = MIRBuilder.buildSub(Dst: IntTy, Src0: V, Src1: OneC);
1376 auto SubnormalRes = assignSPIRVTy(
1377 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: VMinusOne,
1378 Op1: buildSPIRVConstant(IntTy, AllOneMantissa)));
1379 if (PartialCheck == fcNegSubnormal)
1380 SubnormalRes = MIRBuilder.buildAnd(Dst: DstTy, Src0: SubnormalRes, Src1: Sign);
1381 appendToRes(std::move(SubnormalRes));
1382 }
1383
1384 if (FPClassTest PartialCheck = Mask & fcInf) {
1385 if (PartialCheck == fcPosInf)
1386 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
1387 Op0: AsInt, Op1: InfC));
1388 else if (PartialCheck == fcInf)
1389 appendToRes(
1390 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy, Op0: Abs, Op1: InfC));
1391 else { // fcNegInf
1392 APInt NegInf = APFloat::getInf(Sem: Semantics, Negative: true).bitcastToAPInt();
1393 auto NegInfC = buildSPIRVConstant(IntTy, NegInf);
1394 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_EQ, Res: DstTy,
1395 Op0: AsInt, Op1: NegInfC));
1396 }
1397 }
1398
1399 if (FPClassTest PartialCheck = Mask & fcNan) {
1400 auto InfWithQnanBitC =
1401 buildSPIRVConstant(IntTy, std::move(Inf) | QNaNBitMask);
1402 if (PartialCheck == fcNan) {
1403 // isnan(V) ==> abs(V) u> int(inf)
1404 appendToRes(
1405 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_UGT, Res: DstTy, Op0: Abs, Op1: InfC));
1406 } else if (PartialCheck == fcQNan) {
1407 // isquiet(V) ==> abs(V) u>= (unsigned(Inf) | quiet_bit)
1408 appendToRes(MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_UGE, Res: DstTy, Op0: Abs,
1409 Op1: InfWithQnanBitC));
1410 } else { // fcSNan
1411 // issignaling(V) ==> abs(V) u> unsigned(Inf) &&
1412 // abs(V) u< (unsigned(Inf) | quiet_bit)
1413 auto IsNan = assignSPIRVTy(
1414 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_UGT, Res: DstTy, Op0: Abs, Op1: InfC));
1415 auto IsNotQnan = assignSPIRVTy(MIRBuilder.buildICmp(
1416 Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: Abs, Op1: InfWithQnanBitC));
1417 appendToRes(MIRBuilder.buildAnd(Dst: DstTy, Src0: IsNan, Src1: IsNotQnan));
1418 }
1419 }
1420
1421 if (FPClassTest PartialCheck = Mask & fcNormal) {
1422 // isnormal(V) ==> (0 u< exp u< max_exp) ==> (unsigned(exp-1) u<
1423 // (max_exp-1))
1424 APInt ExpLSB = ExpMask & ~(ExpMask.shl(shiftAmt: 1));
1425 auto ExpMinusOne = assignSPIRVTy(
1426 MIRBuilder.buildSub(Dst: IntTy, Src0: Abs, Src1: buildSPIRVConstant(IntTy, ExpLSB)));
1427 APInt MaxExpMinusOne = std::move(ExpMask) - ExpLSB;
1428 auto NormalRes = assignSPIRVTy(
1429 MIRBuilder.buildICmp(Pred: CmpInst::Predicate::ICMP_ULT, Res: DstTy, Op0: ExpMinusOne,
1430 Op1: buildSPIRVConstant(IntTy, MaxExpMinusOne)));
1431 if (PartialCheck == fcNegNormal)
1432 NormalRes = MIRBuilder.buildAnd(Dst: DstTy, Src0: NormalRes, Src1: Sign);
1433 else if (PartialCheck == fcPosNormal) {
1434 auto PosSign = assignSPIRVTy(MIRBuilder.buildXor(
1435 Dst: DstTy, Src0: Sign, Src1: buildSPIRVConstant(DstTy, InversionMask)));
1436 NormalRes = MIRBuilder.buildAnd(Dst: DstTy, Src0: NormalRes, Src1: PosSign);
1437 }
1438 appendToRes(std::move(NormalRes));
1439 }
1440
1441 MIRBuilder.buildCopy(Res: DstReg, Op: Res);
1442 MI.eraseFromParent();
1443 return true;
1444}
1445