1//===- X86LegalizerInfo.cpp --------------------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the Machinelegalizer class for X86.
10/// \todo This should be generated by TableGen.
11//===----------------------------------------------------------------------===//
12
13#include "X86LegalizerInfo.h"
14#include "X86Subtarget.h"
15#include "X86TargetMachine.h"
16#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
17#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
18#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
19#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
20#include "llvm/CodeGen/MachineConstantPool.h"
21#include "llvm/CodeGen/MachineFrameInfo.h"
22#include "llvm/CodeGen/TargetOpcodes.h"
23#include "llvm/CodeGen/ValueTypes.h"
24#include "llvm/IR/DerivedTypes.h"
25#include "llvm/IR/IntrinsicsX86.h"
26#include "llvm/IR/Type.h"
27
28using namespace llvm;
29using namespace MIPatternMatch;
30using namespace TargetOpcode;
31using namespace LegalizeActions;
32using namespace LegalityPredicates;
33
34X86LegalizerInfo::X86LegalizerInfo(const X86Subtarget &STI,
35 const X86TargetMachine &TM)
36 : Subtarget(STI) {
37
38 bool Is64Bit = Subtarget.is64Bit();
39 bool HasCMOV = Subtarget.canUseCMOV();
40 bool HasSSE1 = Subtarget.hasSSE1();
41 bool HasSSE2 = Subtarget.hasSSE2();
42 bool HasSSE41 = Subtarget.hasSSE41();
43 bool HasAVX = Subtarget.hasAVX();
44 bool HasAVX2 = Subtarget.hasAVX2();
45 bool HasAVX512 = Subtarget.hasAVX512();
46 bool HasVLX = Subtarget.hasVLX();
47 bool HasDQI = Subtarget.hasAVX512() && Subtarget.hasDQI();
48 bool HasBWI = Subtarget.hasAVX512() && Subtarget.hasBWI();
49 bool UseX87 = !Subtarget.useSoftFloat() && Subtarget.hasX87();
50 bool HasPOPCNT = Subtarget.hasPOPCNT();
51 bool HasLZCNT = Subtarget.hasLZCNT();
52 bool HasBMI = Subtarget.hasBMI();
53
54 const LLT p0 = LLT::pointer(AddressSpace: 0, SizeInBits: TM.getPointerSizeInBits(AS: 0));
55 const LLT s1 = LLT::scalar(SizeInBits: 1);
56 const LLT s8 = LLT::scalar(SizeInBits: 8);
57 const LLT s16 = LLT::scalar(SizeInBits: 16);
58 const LLT s32 = LLT::scalar(SizeInBits: 32);
59 const LLT s64 = LLT::scalar(SizeInBits: 64);
60 const LLT s80 = LLT::scalar(SizeInBits: 80);
61 const LLT s128 = LLT::scalar(SizeInBits: 128);
62 const LLT sMaxScalar = Subtarget.is64Bit() ? s64 : s32;
63 const LLT v2s32 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32);
64 const LLT v4s8 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 8);
65
66 const LLT v16s8 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 8);
67 const LLT v8s16 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16);
68 const LLT v4s32 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32);
69 const LLT v2s64 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64);
70 const LLT v2p0 = LLT::fixed_vector(NumElements: 2, ScalarTy: p0);
71
72 const LLT v32s8 = LLT::fixed_vector(NumElements: 32, ScalarSizeInBits: 8);
73 const LLT v16s16 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 16);
74 const LLT v8s32 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 32);
75 const LLT v4s64 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 64);
76 const LLT v4p0 = LLT::fixed_vector(NumElements: 4, ScalarTy: p0);
77
78 const LLT v64s8 = LLT::fixed_vector(NumElements: 64, ScalarSizeInBits: 8);
79 const LLT v32s16 = LLT::fixed_vector(NumElements: 32, ScalarSizeInBits: 16);
80 const LLT v16s32 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 32);
81 const LLT v8s64 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 64);
82
83 const LLT s8MaxVector = HasAVX512 ? v64s8 : HasAVX ? v32s8 : v16s8;
84 const LLT s16MaxVector = HasAVX512 ? v32s16 : HasAVX ? v16s16 : v8s16;
85 const LLT s32MaxVector = HasAVX512 ? v16s32 : HasAVX ? v8s32 : v4s32;
86 const LLT s64MaxVector = HasAVX512 ? v8s64 : HasAVX ? v4s64 : v2s64;
87
88 // todo: AVX512 bool vector predicate types
89
90 // implicit/constants
91 // 32/64-bits needs support for s64/s128 to handle cases:
92 // s64 = EXTEND (G_IMPLICIT_DEF s32) -> s64 = G_IMPLICIT_DEF
93 // s128 = EXTEND (G_IMPLICIT_DEF s32/s64) -> s128 = G_IMPLICIT_DEF
94 getActionDefinitionsBuilder(
95 Opcodes: {G_IMPLICIT_DEF, G_PHI, G_FREEZE, G_CONSTANT_FOLD_BARRIER})
96 .legalFor(Types: {p0, s1, s8, s16, s32, s64})
97 .legalFor(Pred: UseX87, Types: {s80})
98 .legalFor(Pred: Is64Bit, Types: {s128})
99 .legalFor(Pred: HasSSE2, Types: {v16s8, v8s16, v4s32, v2s64})
100 .legalFor(Pred: HasAVX, Types: {v32s8, v16s16, v8s32, v4s64})
101 .legalFor(Pred: HasAVX512, Types: {v64s8, v32s16, v16s32, v8s64})
102 .widenScalarOrEltToNextPow2(TypeIdx: 0, /*Min=*/MinSize: 8)
103 .clampScalarOrElt(TypeIdx: 0, MinTy: s8, MaxTy: sMaxScalar)
104 .moreElementsToNextPow2(TypeIdx: 0)
105 .clampNumElements(TypeIdx: 0, MinTy: v16s8, MaxTy: s8MaxVector)
106 .clampNumElements(TypeIdx: 0, MinTy: v8s16, MaxTy: s16MaxVector)
107 .clampNumElements(TypeIdx: 0, MinTy: v4s32, MaxTy: s32MaxVector)
108 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: s64MaxVector)
109 .clampMaxNumElements(TypeIdx: 0, EltTy: p0,
110 MaxElements: Is64Bit ? s64MaxVector.getNumElements()
111 : s32MaxVector.getNumElements())
112 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0);
113
114 getActionDefinitionsBuilder(Opcode: G_CONSTANT)
115 .legalFor(Types: {p0, s8, s16, s32})
116 .legalFor(Pred: Is64Bit, Types: {s64})
117 .widenScalarToNextPow2(TypeIdx: 0, /*Min=*/MinSize: 8)
118 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: sMaxScalar);
119
120 getActionDefinitionsBuilder(Opcodes: {G_LROUND, G_LLROUND})
121 .widenScalarIf(Predicate: typeIs(TypeIdx: 1, TypesInit: s16),
122 Mutation: [=](const LegalityQuery &) {
123 return std::pair<unsigned, LLT>(1, s32);
124 })
125 .libcall();
126
127 getActionDefinitionsBuilder(
128 Opcodes: {G_FCOS, G_FCOSH, G_FACOS, G_FSIN, G_FSINH, G_FASIN, G_FTAN,
129 G_FTANH, G_FATAN, G_FATAN2, G_FPOW, G_FEXP, G_FEXP2, G_FEXP10,
130 G_FLOG, G_FLOG2, G_FLOG10, G_FPOWI, G_FSINCOS, G_FCEIL, G_FFLOOR})
131 .libcall();
132
133 getActionDefinitionsBuilder(Opcode: G_FNEG)
134 .legalFor(Pred: UseX87 && !HasSSE1, Types: {s32})
135 .legalFor(Pred: UseX87 && !HasSSE2, Types: {s64})
136 .legalFor(Pred: UseX87, Types: {s80})
137 .lower();
138
139 getActionDefinitionsBuilder(Opcode: G_FSQRT)
140 .legalFor(Pred: HasSSE1 || UseX87, Types: {s32})
141 .legalFor(Pred: HasSSE2 || UseX87, Types: {s64})
142 .legalFor(Pred: UseX87, Types: {s80});
143
144 getActionDefinitionsBuilder(Opcodes: {G_GET_ROUNDING, G_SET_ROUNDING})
145 .customFor(Types: {s32});
146
147 // merge/unmerge
148 for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
149 unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;
150 unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;
151 getActionDefinitionsBuilder(Opcode: Op)
152 .widenScalarToNextPow2(TypeIdx: LitTyIdx, /*Min=*/MinSize: 8)
153 .widenScalarToNextPow2(TypeIdx: BigTyIdx, /*Min=*/MinSize: 16)
154 .minScalar(TypeIdx: LitTyIdx, Ty: s8)
155 .minScalar(TypeIdx: BigTyIdx, Ty: s32)
156 .legalIf(Predicate: [=](const LegalityQuery &Q) {
157 switch (Q.Types[BigTyIdx].getSizeInBits()) {
158 case 16:
159 case 32:
160 case 64:
161 case 128:
162 case 256:
163 case 512:
164 break;
165 default:
166 return false;
167 }
168 switch (Q.Types[LitTyIdx].getSizeInBits()) {
169 case 8:
170 case 16:
171 case 32:
172 case 64:
173 case 128:
174 case 256:
175 return true;
176 default:
177 return false;
178 }
179 });
180 }
181
182 getActionDefinitionsBuilder(Opcodes: {G_UMIN, G_UMAX, G_SMIN, G_SMAX})
183 .widenScalarToNextPow2(TypeIdx: 0, /*Min=*/MinSize: 32)
184 .lower();
185
186 // integer addition/subtraction
187 getActionDefinitionsBuilder(Opcodes: {G_ADD, G_SUB})
188 .legalFor(Types: {s8, s16, s32})
189 .legalFor(Pred: Is64Bit, Types: {s64})
190 .legalFor(Pred: HasSSE2, Types: {v16s8, v8s16, v4s32, v2s64})
191 .legalFor(Pred: HasAVX2, Types: {v32s8, v16s16, v8s32, v4s64})
192 .legalFor(Pred: HasAVX512, Types: {v16s32, v8s64})
193 .legalFor(Pred: HasBWI, Types: {v64s8, v32s16})
194 .clampMinNumElements(TypeIdx: 0, EltTy: s8, MinElements: 16)
195 .clampMinNumElements(TypeIdx: 0, EltTy: s16, MinElements: 8)
196 .clampMinNumElements(TypeIdx: 0, EltTy: s32, MinElements: 4)
197 .clampMinNumElements(TypeIdx: 0, EltTy: s64, MinElements: 2)
198 .clampMaxNumElements(TypeIdx: 0, EltTy: s8, MaxElements: HasBWI ? 64 : (HasAVX2 ? 32 : 16))
199 .clampMaxNumElements(TypeIdx: 0, EltTy: s16, MaxElements: HasBWI ? 32 : (HasAVX2 ? 16 : 8))
200 .clampMaxNumElements(TypeIdx: 0, EltTy: s32, MaxElements: HasAVX512 ? 16 : (HasAVX2 ? 8 : 4))
201 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: HasAVX512 ? 8 : (HasAVX2 ? 4 : 2))
202 .widenScalarToNextPow2(TypeIdx: 0, /*Min=*/MinSize: 32)
203 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: sMaxScalar)
204 .scalarize(TypeIdx: 0);
205
206 getActionDefinitionsBuilder(Opcodes: {G_UADDE, G_UADDO, G_USUBE, G_USUBO})
207 .legalFor(Types: {{s8, s8}, {s16, s8}, {s32, s8}})
208 .legalFor(Pred: Is64Bit, Types: {{s64, s8}})
209 .widenScalarToNextPow2(TypeIdx: 0, /*Min=*/MinSize: 32)
210 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: sMaxScalar)
211 .clampScalar(TypeIdx: 1, MinTy: s8, MaxTy: s8)
212 .scalarize(TypeIdx: 0);
213
214 // integer multiply
215 getActionDefinitionsBuilder(Opcode: G_MUL)
216 .legalFor(Types: {s8, s16, s32})
217 .legalFor(Pred: Is64Bit, Types: {s64})
218 .legalFor(Pred: HasSSE2, Types: {v8s16})
219 .legalFor(Pred: HasSSE41, Types: {v4s32})
220 .legalFor(Pred: HasAVX2, Types: {v16s16, v8s32})
221 .legalFor(Pred: HasAVX512, Types: {v16s32})
222 .legalFor(Pred: HasDQI, Types: {v8s64})
223 .legalFor(Pred: HasDQI && HasVLX, Types: {v2s64, v4s64})
224 .legalFor(Pred: HasBWI, Types: {v32s16})
225 .clampMinNumElements(TypeIdx: 0, EltTy: s16, MinElements: 8)
226 .clampMinNumElements(TypeIdx: 0, EltTy: s32, MinElements: 4)
227 .clampMinNumElements(TypeIdx: 0, EltTy: s64, MinElements: HasVLX ? 2 : 8)
228 .clampMaxNumElements(TypeIdx: 0, EltTy: s16, MaxElements: HasBWI ? 32 : (HasAVX2 ? 16 : 8))
229 .clampMaxNumElements(TypeIdx: 0, EltTy: s32, MaxElements: HasAVX512 ? 16 : (HasAVX2 ? 8 : 4))
230 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 8)
231 .widenScalarToNextPow2(TypeIdx: 0, /*Min=*/MinSize: 32)
232 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: sMaxScalar)
233 .scalarize(TypeIdx: 0);
234
235 getActionDefinitionsBuilder(Opcodes: {G_SMULH, G_UMULH})
236 .legalFor(Types: {s8, s16, s32})
237 .legalFor(Pred: Is64Bit, Types: {s64})
238 .widenScalarToNextPow2(TypeIdx: 0, /*Min=*/MinSize: 32)
239 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: sMaxScalar)
240 .scalarize(TypeIdx: 0);
241
242 // integer divisions
243 getActionDefinitionsBuilder(Opcodes: {G_SDIV, G_SREM, G_UDIV, G_UREM})
244 .legalFor(Types: {s8, s16, s32})
245 .legalFor(Pred: Is64Bit, Types: {s64})
246 .libcallFor(Types: {s64})
247 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: sMaxScalar);
248
249 // integer shifts
250 getActionDefinitionsBuilder(Opcodes: {G_SHL, G_LSHR, G_ASHR})
251 .legalFor(Types: {{s8, s8}, {s16, s8}, {s32, s8}})
252 .legalFor(Pred: Is64Bit, Types: {{s64, s8}})
253 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: sMaxScalar)
254 .clampScalar(TypeIdx: 1, MinTy: s8, MaxTy: s8);
255
256 // integer logic
257 getActionDefinitionsBuilder(Opcodes: {G_AND, G_OR, G_XOR})
258 .legalFor(Types: {s8, s16, s32})
259 .legalFor(Pred: Is64Bit, Types: {s64})
260 .legalFor(Pred: HasSSE2, Types: {v16s8, v8s16, v4s32, v2s64})
261 .legalFor(Pred: HasAVX, Types: {v32s8, v16s16, v8s32, v4s64})
262 .legalFor(Pred: HasAVX512, Types: {v64s8, v32s16, v16s32, v8s64})
263 .clampNumElements(TypeIdx: 0, MinTy: v16s8, MaxTy: s8MaxVector)
264 .clampNumElements(TypeIdx: 0, MinTy: v8s16, MaxTy: s16MaxVector)
265 .clampNumElements(TypeIdx: 0, MinTy: v4s32, MaxTy: s32MaxVector)
266 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: s64MaxVector)
267 .widenScalarToNextPow2(TypeIdx: 0, /*Min=*/MinSize: 32)
268 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: sMaxScalar)
269 .scalarize(TypeIdx: 0);
270
271 // integer comparison
272 const std::initializer_list<LLT> IntTypes32 = {s8, s16, s32, p0};
273 const std::initializer_list<LLT> IntTypes64 = {s8, s16, s32, s64, p0};
274
275 getActionDefinitionsBuilder(Opcode: G_ICMP)
276 .legalForCartesianProduct(Types0: {s8}, Types1: Is64Bit ? IntTypes64 : IntTypes32)
277 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: s8)
278 .widenScalarToNextPow2(TypeIdx: 1, /*Min=*/MinSize: 8)
279 .clampScalar(TypeIdx: 1, MinTy: s8, MaxTy: sMaxScalar);
280
281 // bswap
282 getActionDefinitionsBuilder(Opcode: G_BSWAP)
283 .legalFor(Types: {s32})
284 .legalFor(Pred: Is64Bit, Types: {s64})
285 .widenScalarToNextPow2(TypeIdx: 0, /*Min=*/MinSize: 32)
286 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: sMaxScalar);
287
288 // popcount
289 getActionDefinitionsBuilder(Opcode: G_CTPOP)
290 .legalFor(Pred: HasPOPCNT, Types: {{s16, s16}, {s32, s32}})
291 .legalFor(Pred: HasPOPCNT && Is64Bit, Types: {{s64, s64}})
292 .widenScalarToNextPow2(TypeIdx: 1, /*Min=*/MinSize: 16)
293 .clampScalar(TypeIdx: 1, MinTy: s16, MaxTy: sMaxScalar)
294 .scalarSameSizeAs(TypeIdx: 0, SameSizeIdx: 1);
295
296 // count leading zeros (LZCNT)
297 getActionDefinitionsBuilder(Opcode: G_CTLZ)
298 .legalFor(Pred: HasLZCNT, Types: {{s16, s16}, {s32, s32}})
299 .legalFor(Pred: HasLZCNT && Is64Bit, Types: {{s64, s64}})
300 .widenScalarToNextPow2(TypeIdx: 1, /*Min=*/MinSize: 16)
301 .clampScalar(TypeIdx: 1, MinTy: s16, MaxTy: sMaxScalar)
302 .scalarSameSizeAs(TypeIdx: 0, SameSizeIdx: 1);
303
304 // count trailing zeros
305 getActionDefinitionsBuilder(Opcode: G_CTTZ_ZERO_POISON)
306 .legalFor(Types: {{s16, s16}, {s32, s32}})
307 .legalFor(Pred: Is64Bit, Types: {{s64, s64}})
308 .widenScalarToNextPow2(TypeIdx: 1, /*Min=*/MinSize: 16)
309 .clampScalar(TypeIdx: 1, MinTy: s16, MaxTy: sMaxScalar)
310 .scalarSameSizeAs(TypeIdx: 0, SameSizeIdx: 1);
311
312 getActionDefinitionsBuilder(Opcode: G_CTTZ)
313 .legalFor(Pred: HasBMI, Types: {{s16, s16}, {s32, s32}})
314 .legalFor(Pred: HasBMI && Is64Bit, Types: {{s64, s64}})
315 .widenScalarToNextPow2(TypeIdx: 1, /*Min=*/MinSize: 16)
316 .clampScalar(TypeIdx: 1, MinTy: s16, MaxTy: sMaxScalar)
317 .scalarSameSizeAs(TypeIdx: 0, SameSizeIdx: 1);
318
319 getActionDefinitionsBuilder(Opcode: G_BR).alwaysLegal();
320 getActionDefinitionsBuilder(Opcode: G_BRCOND).legalFor(Types: {s1});
321
322 // pointer handling
323 const std::initializer_list<LLT> PtrTypes32 = {s1, s8, s16, s32};
324 const std::initializer_list<LLT> PtrTypes64 = {s1, s8, s16, s32, s64};
325
326 getActionDefinitionsBuilder(Opcode: G_PTRTOINT)
327 .legalForCartesianProduct(Types0: Is64Bit ? PtrTypes64 : PtrTypes32, Types1: {p0})
328 .maxScalar(TypeIdx: 0, Ty: sMaxScalar)
329 .widenScalarToNextPow2(TypeIdx: 0, /*Min*/ MinSize: 8);
330
331 getActionDefinitionsBuilder(Opcode: G_INTTOPTR).legalFor(Types: {{p0, sMaxScalar}});
332
333 getActionDefinitionsBuilder(Opcode: G_CONSTANT_POOL).legalFor(Types: {p0});
334
335 getActionDefinitionsBuilder(Opcode: G_PTR_ADD)
336 .legalFor(Types: {{p0, s32}})
337 .legalFor(Pred: Is64Bit, Types: {{p0, s64}})
338 .widenScalarToNextPow2(TypeIdx: 1, /*Min*/ MinSize: 32)
339 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: sMaxScalar);
340
341 getActionDefinitionsBuilder(Opcode: G_FRAME_INDEX).legalFor(Types: {p0});
342
343 getActionDefinitionsBuilder(Opcode: G_GLOBAL_VALUE).customFor(Types: {p0});
344
345 // load/store: add more corner cases
346 for (unsigned Op : {G_LOAD, G_STORE}) {
347 auto &Action = getActionDefinitionsBuilder(Opcode: Op);
348 Action.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: s8, .Type1: p0, .MemTy: s8, .Align: 1},
349 {.Type0: s16, .Type1: p0, .MemTy: s16, .Align: 1},
350 {.Type0: s32, .Type1: p0, .MemTy: s32, .Align: 1},
351 {.Type0: s80, .Type1: p0, .MemTy: s80, .Align: 1},
352 {.Type0: p0, .Type1: p0, .MemTy: p0, .Align: 1},
353 {.Type0: v4s8, .Type1: p0, .MemTy: v4s8, .Align: 1}});
354 if (Is64Bit)
355 Action.legalForTypesWithMemDesc(
356 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s64, .Align: 1}, {.Type0: v2s32, .Type1: p0, .MemTy: v2s32, .Align: 1}});
357
358 if (HasSSE1)
359 Action.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: v4s32, .Type1: p0, .MemTy: v4s32, .Align: 1}});
360 if (HasSSE2)
361 Action.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: v16s8, .Type1: p0, .MemTy: v16s8, .Align: 1},
362 {.Type0: v8s16, .Type1: p0, .MemTy: v8s16, .Align: 1},
363 {.Type0: v2s64, .Type1: p0, .MemTy: v2s64, .Align: 1},
364 {.Type0: v2p0, .Type1: p0, .MemTy: v2p0, .Align: 1}});
365 if (HasAVX)
366 Action.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: v32s8, .Type1: p0, .MemTy: v32s8, .Align: 1},
367 {.Type0: v16s16, .Type1: p0, .MemTy: v16s16, .Align: 1},
368 {.Type0: v8s32, .Type1: p0, .MemTy: v8s32, .Align: 1},
369 {.Type0: v4s64, .Type1: p0, .MemTy: v4s64, .Align: 1},
370 {.Type0: v4p0, .Type1: p0, .MemTy: v4p0, .Align: 1}});
371 if (HasAVX512)
372 Action.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: v64s8, .Type1: p0, .MemTy: v64s8, .Align: 1},
373 {.Type0: v32s16, .Type1: p0, .MemTy: v32s16, .Align: 1},
374 {.Type0: v16s32, .Type1: p0, .MemTy: v16s32, .Align: 1},
375 {.Type0: v8s64, .Type1: p0, .MemTy: v8s64, .Align: 1}});
376
377 // X86 supports extending loads but not stores for GPRs
378 if (Op == G_LOAD) {
379 Action.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: s8, .Type1: p0, .MemTy: s1, .Align: 1},
380 {.Type0: s16, .Type1: p0, .MemTy: s8, .Align: 1},
381 {.Type0: s32, .Type1: p0, .MemTy: s8, .Align: 1},
382 {.Type0: s32, .Type1: p0, .MemTy: s16, .Align: 1}});
383 if (Is64Bit)
384 Action.legalForTypesWithMemDesc(
385 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s8, .Align: 1}, {.Type0: s64, .Type1: p0, .MemTy: s16, .Align: 1}, {.Type0: s64, .Type1: p0, .MemTy: s32, .Align: 1}});
386 } else {
387 Action.customIf(Predicate: [=](const LegalityQuery &Query) {
388 return Query.Types[0] != Query.MMODescrs[0].MemoryTy;
389 });
390 }
391 Action.widenScalarToNextPow2(TypeIdx: 0, /*Min=*/MinSize: 8)
392 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: sMaxScalar)
393 .scalarize(TypeIdx: 0);
394 }
395
396 for (unsigned Op : {G_SEXTLOAD, G_ZEXTLOAD}) {
397 auto &Action = getActionDefinitionsBuilder(Opcode: Op);
398 Action.legalForTypesWithMemDesc(
399 TypesAndMemDesc: {{.Type0: s16, .Type1: p0, .MemTy: s8, .Align: 1}, {.Type0: s32, .Type1: p0, .MemTy: s8, .Align: 1}, {.Type0: s32, .Type1: p0, .MemTy: s16, .Align: 1}});
400 if (Is64Bit)
401 Action.legalForTypesWithMemDesc(
402 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s8, .Align: 1}, {.Type0: s64, .Type1: p0, .MemTy: s16, .Align: 1}, {.Type0: s64, .Type1: p0, .MemTy: s32, .Align: 1}});
403 // TODO - SSE41/AVX2/AVX512F/AVX512BW vector extensions
404 }
405
406 for (unsigned Op : {G_FPEXTLOAD, G_FPTRUNCSTORE}) {
407 auto &Action = getActionDefinitionsBuilder(Opcode: Op);
408 Action.legalForTypesWithMemDesc(
409 Pred: UseX87, TypesAndMemDesc: {{.Type0: s80, .Type1: p0, .MemTy: s32, .Align: 1}, {.Type0: s80, .Type1: p0, .MemTy: s64, .Align: 1}, {.Type0: s64, .Type1: p0, .MemTy: s32, .Align: 1}});
410 }
411
412 // sext, zext, and anyext
413 getActionDefinitionsBuilder(Opcode: G_ANYEXT)
414 .legalFor(Types: {s8, s16, s32, s128})
415 .legalFor(Pred: Is64Bit, Types: {s64})
416 .widenScalarToNextPow2(TypeIdx: 0, /*Min=*/MinSize: 8)
417 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: sMaxScalar)
418 .widenScalarToNextPow2(TypeIdx: 1, /*Min=*/MinSize: 8)
419 .clampScalar(TypeIdx: 1, MinTy: s8, MaxTy: sMaxScalar)
420 .scalarize(TypeIdx: 0);
421
422 getActionDefinitionsBuilder(Opcodes: {G_SEXT, G_ZEXT})
423 .legalFor(Types: {s8, s16, s32})
424 .legalFor(Pred: Is64Bit, Types: {s64})
425 .widenScalarToNextPow2(TypeIdx: 0, /*Min=*/MinSize: 8)
426 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: sMaxScalar)
427 .widenScalarToNextPow2(TypeIdx: 1, /*Min=*/MinSize: 8)
428 .clampScalar(TypeIdx: 1, MinTy: s8, MaxTy: sMaxScalar)
429 .scalarize(TypeIdx: 0);
430
431 getActionDefinitionsBuilder(Opcode: G_TRUNC).legalForCartesianProduct(
432 Types0: {s1, s8, s16, s32, s64}, Types1: {s8, s16, s32, s64, s128});
433
434 getActionDefinitionsBuilder(Opcode: G_SEXT_INREG).lower();
435
436 // fp constants
437 getActionDefinitionsBuilder(Opcode: G_FCONSTANT)
438 .legalFor(Types: {s32, s64})
439 .legalFor(Pred: UseX87, Types: {s80});
440
441 // fp arithmetic
442 getActionDefinitionsBuilder(Opcodes: {G_FADD, G_FSUB, G_FMUL, G_FDIV})
443 .legalFor(Types: {s32, s64})
444 .legalFor(Pred: HasSSE1, Types: {v4s32})
445 .legalFor(Pred: HasSSE2, Types: {v2s64})
446 .legalFor(Pred: HasAVX, Types: {v8s32, v4s64})
447 .legalFor(Pred: HasAVX512, Types: {v16s32, v8s64})
448 .legalFor(Pred: UseX87, Types: {s80});
449
450 getActionDefinitionsBuilder(Opcode: G_FABS)
451 .legalFor(Pred: UseX87, Types: {s80})
452 .legalFor(Pred: UseX87 && !Is64Bit, Types: {s64})
453 .lower();
454
455 // fp comparison
456 getActionDefinitionsBuilder(Opcode: G_FCMP)
457 .legalFor(Pred: HasSSE1 || UseX87, Types: {s8, s32})
458 .legalFor(Pred: HasSSE2 || UseX87, Types: {s8, s64})
459 .legalFor(Pred: UseX87, Types: {s8, s80})
460 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: s8)
461 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: HasSSE2 ? s64 : s32)
462 .widenScalarToNextPow2(TypeIdx: 1);
463
464 // fp conversions
465 getActionDefinitionsBuilder(Opcode: G_FPEXT)
466 .legalFor(Pred: HasSSE2, Types: {{s64, s32}})
467 .legalFor(Pred: HasAVX, Types: {{v4s64, v4s32}})
468 .legalFor(Pred: HasAVX512, Types: {{v8s64, v8s32}})
469 .lowerFor(Pred: UseX87, Types: {{s64, s32}, {s80, s32}, {s80, s64}})
470 .libcall();
471
472 getActionDefinitionsBuilder(Opcode: G_FPTRUNC)
473 .legalFor(Pred: HasSSE2, Types: {{s32, s64}})
474 .legalFor(Pred: HasAVX, Types: {{v4s32, v4s64}})
475 .legalFor(Pred: HasAVX512, Types: {{v8s32, v8s64}})
476 .lowerFor(Pred: UseX87, Types: {{s32, s64}, {s32, s80}, {s64, s80}});
477
478 getActionDefinitionsBuilder(Opcode: G_SITOFP)
479 .legalFor(Pred: HasSSE1, Types: {{s32, s32}})
480 .legalFor(Pred: HasSSE1 && Is64Bit, Types: {{s32, s64}})
481 .legalFor(Pred: HasSSE2, Types: {{s64, s32}})
482 .legalFor(Pred: HasSSE2 && Is64Bit, Types: {{s64, s64}})
483 .clampScalar(TypeIdx: 1, MinTy: (UseX87 && !HasSSE1) ? s16 : s32, MaxTy: sMaxScalar)
484 .widenScalarToNextPow2(TypeIdx: 1)
485 .customForCartesianProduct(Pred: UseX87, Types0: {s32, s64, s80}, Types1: {s16, s32, s64})
486 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: HasSSE2 ? s64 : s32)
487 .widenScalarToNextPow2(TypeIdx: 0);
488
489 getActionDefinitionsBuilder(Opcode: G_FPTOSI)
490 .legalFor(Pred: HasSSE1, Types: {{s32, s32}})
491 .legalFor(Pred: HasSSE1 && Is64Bit, Types: {{s64, s32}})
492 .legalFor(Pred: HasSSE2, Types: {{s32, s64}})
493 .legalFor(Pred: HasSSE2 && Is64Bit, Types: {{s64, s64}})
494 .clampScalar(TypeIdx: 0, MinTy: (UseX87 && !HasSSE1) ? s16 : s32, MaxTy: sMaxScalar)
495 .widenScalarToNextPow2(TypeIdx: 0)
496 .customForCartesianProduct(Pred: UseX87, Types0: {s16, s32, s64}, Types1: {s32, s64, s80})
497 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: HasSSE2 ? s64 : s32)
498 .widenScalarToNextPow2(TypeIdx: 1);
499
500 // For G_UITOFP and G_FPTOUI without AVX512, we have to custom legalize types
501 // <= s32 manually. Otherwise, in custom handler there is no way to
502 // understand whether s32 is an original type and we need to promote it to
503 // s64 or s32 is obtained after widening and we shouldn't widen it to s64.
504 //
505 // For AVX512 we simply widen types as there is direct mapping from opcodes
506 // to asm instructions.
507 getActionDefinitionsBuilder(Opcode: G_UITOFP)
508 .legalFor(Pred: HasAVX512, Types: {{s32, s32}, {s32, s64}, {s64, s32}, {s64, s64}})
509 .customIf(Predicate: [=](const LegalityQuery &Query) {
510 return !HasAVX512 &&
511 ((HasSSE1 && typeIs(TypeIdx: 0, TypesInit: s32)(Query)) ||
512 (HasSSE2 && typeIs(TypeIdx: 0, TypesInit: s64)(Query))) &&
513 scalarNarrowerThan(TypeIdx: 1, Size: Is64Bit ? 64 : 32)(Query);
514 })
515 .lowerIf(Predicate: [=](const LegalityQuery &Query) {
516 // Lower conversions from s64
517 return !HasAVX512 &&
518 ((HasSSE1 && typeIs(TypeIdx: 0, TypesInit: s32)(Query)) ||
519 (HasSSE2 && typeIs(TypeIdx: 0, TypesInit: s64)(Query))) &&
520 (Is64Bit && typeIs(TypeIdx: 1, TypesInit: s64)(Query));
521 })
522 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: HasSSE2 ? s64 : s32)
523 .widenScalarToNextPow2(TypeIdx: 0)
524 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: sMaxScalar)
525 .widenScalarToNextPow2(TypeIdx: 1);
526
527 getActionDefinitionsBuilder(Opcode: G_FPTOUI)
528 .legalFor(Pred: HasAVX512, Types: {{s32, s32}, {s32, s64}, {s64, s32}, {s64, s64}})
529 .customIf(Predicate: [=](const LegalityQuery &Query) {
530 return !HasAVX512 &&
531 ((HasSSE1 && typeIs(TypeIdx: 1, TypesInit: s32)(Query)) ||
532 (HasSSE2 && typeIs(TypeIdx: 1, TypesInit: s64)(Query))) &&
533 scalarNarrowerThan(TypeIdx: 0, Size: Is64Bit ? 64 : 32)(Query);
534 })
535 // TODO: replace with customized legalization using
536 // specifics of cvttsd2si. The selection of this node requires
537 // a vector type. Either G_SCALAR_TO_VECTOR is needed or more advanced
538 // support of G_BUILD_VECTOR/G_INSERT_VECTOR_ELT is required beforehand.
539 .lowerIf(Predicate: [=](const LegalityQuery &Query) {
540 return !HasAVX512 &&
541 ((HasSSE1 && typeIs(TypeIdx: 1, TypesInit: s32)(Query)) ||
542 (HasSSE2 && typeIs(TypeIdx: 1, TypesInit: s64)(Query))) &&
543 (Is64Bit && typeIs(TypeIdx: 0, TypesInit: s64)(Query));
544 })
545 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: sMaxScalar)
546 .widenScalarToNextPow2(TypeIdx: 0)
547 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: HasSSE2 ? s64 : s32)
548 .widenScalarToNextPow2(TypeIdx: 1);
549
550 // vector ops
551 getActionDefinitionsBuilder(Opcode: G_BUILD_VECTOR)
552 .customIf(Predicate: [=](const LegalityQuery &Query) {
553 return (HasSSE1 && typeInSet(TypeIdx: 0, TypesInit: {v4s32})(Query)) ||
554 (HasSSE2 && typeInSet(TypeIdx: 0, TypesInit: {v2s64, v8s16, v16s8})(Query)) ||
555 (HasAVX && typeInSet(TypeIdx: 0, TypesInit: {v4s64, v8s32, v16s16, v32s8})(Query)) ||
556 (HasAVX512 &&
557 typeInSet(TypeIdx: 0, TypesInit: {v8s64, v16s32, v32s16, v64s8})(Query));
558 })
559 .clampNumElements(TypeIdx: 0, MinTy: v16s8, MaxTy: s8MaxVector)
560 .clampNumElements(TypeIdx: 0, MinTy: v8s16, MaxTy: s16MaxVector)
561 .clampNumElements(TypeIdx: 0, MinTy: v4s32, MaxTy: s32MaxVector)
562 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: s64MaxVector)
563 .moreElementsToNextPow2(TypeIdx: 0);
564
565 getActionDefinitionsBuilder(Opcodes: {G_EXTRACT, G_INSERT})
566 .legalIf(Predicate: [=](const LegalityQuery &Query) {
567 unsigned SubIdx = Query.Opcode == G_EXTRACT ? 0 : 1;
568 unsigned FullIdx = Query.Opcode == G_EXTRACT ? 1 : 0;
569 return (HasAVX && typePairInSet(TypeIdx0: SubIdx, TypeIdx1: FullIdx,
570 TypesInit: {{v16s8, v32s8},
571 {v8s16, v16s16},
572 {v4s32, v8s32},
573 {v2s64, v4s64}})(Query)) ||
574 (HasAVX512 && typePairInSet(TypeIdx0: SubIdx, TypeIdx1: FullIdx,
575 TypesInit: {{v16s8, v64s8},
576 {v32s8, v64s8},
577 {v8s16, v32s16},
578 {v16s16, v32s16},
579 {v4s32, v16s32},
580 {v8s32, v16s32},
581 {v2s64, v8s64},
582 {v4s64, v8s64}})(Query));
583 });
584
585 // todo: only permit dst types up to max legal vector register size?
586 getActionDefinitionsBuilder(Opcode: G_CONCAT_VECTORS)
587 .legalFor(
588 Pred: HasSSE1,
589 Types: {{v32s8, v16s8}, {v16s16, v8s16}, {v8s32, v4s32}, {v4s64, v2s64}})
590 .legalFor(Pred: HasAVX, Types: {{v64s8, v16s8},
591 {v64s8, v32s8},
592 {v32s16, v8s16},
593 {v32s16, v16s16},
594 {v16s32, v4s32},
595 {v16s32, v8s32},
596 {v8s64, v2s64},
597 {v8s64, v4s64}});
598
599 // todo: vectors and address spaces
600 getActionDefinitionsBuilder(Opcode: G_SELECT)
601 .legalFor(Types: {{s16, s32}, {s32, s32}, {p0, s32}})
602 .legalFor(Pred: !HasCMOV, Types: {{s8, s32}})
603 .legalFor(Pred: Is64Bit, Types: {{s64, s32}})
604 .legalFor(Pred: UseX87, Types: {{s80, s32}})
605 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: s32)
606 .widenScalarToNextPow2(TypeIdx: 0, /*Min=*/MinSize: 8)
607 .clampScalar(TypeIdx: 0, MinTy: HasCMOV ? s16 : s8, MaxTy: sMaxScalar);
608
609 // memory intrinsics
610 getActionDefinitionsBuilder(Opcodes: {G_MEMCPY, G_MEMMOVE, G_MEMSET}).libcall();
611
612 getActionDefinitionsBuilder(Opcodes: {G_DYN_STACKALLOC, G_STACKSAVE, G_STACKRESTORE})
613 .lower();
614
615 // fp intrinsics
616 // fpclass for i686 is disabled for llvm issue #171992
617 getActionDefinitionsBuilder(Opcode: G_IS_FPCLASS)
618 .lowerFor(Pred: Is64Bit, Types: {{s1, s32}, {s1, s64}, {s1, s80}});
619
620 getActionDefinitionsBuilder(Opcodes: {G_INTRINSIC_ROUNDEVEN, G_INTRINSIC_TRUNC})
621 .scalarize(TypeIdx: 0)
622 .minScalar(TypeIdx: 0, Ty: LLT::scalar(SizeInBits: 32))
623 .libcall();
624
625 getActionDefinitionsBuilder(Opcodes: {G_INTRINSIC, G_INTRINSIC_W_SIDE_EFFECTS})
626 .alwaysLegal();
627 getActionDefinitionsBuilder(Opcodes: {G_TRAP, G_DEBUGTRAP, G_UBSANTRAP}).alwaysLegal();
628 getActionDefinitionsBuilder(Opcode: G_INVOKE_REGION_START).alwaysLegal();
629
630 verify(MII: *STI.getInstrInfo());
631}
632
633bool X86LegalizerInfo::legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI,
634 LostDebugLocObserver &LocObserver) const {
635 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
636 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
637 switch (MI.getOpcode()) {
638 default:
639 // No idea what to do.
640 return false;
641 case TargetOpcode::G_BUILD_VECTOR:
642 return legalizeBuildVector(MI, MRI, Helper);
643 case TargetOpcode::G_FPTOUI:
644 return legalizeFPTOUI(MI, MRI, Helper);
645 case TargetOpcode::G_UITOFP:
646 return legalizeUITOFP(MI, MRI, Helper);
647 case TargetOpcode::G_STORE:
648 return legalizeNarrowingStore(MI, MRI, Helper);
649 case TargetOpcode::G_SITOFP:
650 return legalizeSITOFP(MI, MRI, Helper);
651 case TargetOpcode::G_FPTOSI:
652 return legalizeFPTOSI(MI, MRI, Helper);
653 case TargetOpcode::G_GET_ROUNDING:
654 return legalizeGETROUNDING(MI, MRI, Helper);
655 case TargetOpcode::G_SET_ROUNDING:
656 return legalizeSETROUNDING(MI, MRI, Helper);
657 case TargetOpcode::G_GLOBAL_VALUE:
658 return legalizeGLOBAL_VALUE(MI, MRI, Helper);
659 }
660 llvm_unreachable("expected switch to return");
661}
662
663bool X86LegalizerInfo::legalizeSITOFP(MachineInstr &MI,
664 MachineRegisterInfo &MRI,
665 LegalizerHelper &Helper) const {
666 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
667 MachineFunction &MF = *MI.getMF();
668 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
669
670 assert((SrcTy.getSizeInBits() == 16 || SrcTy.getSizeInBits() == 32 ||
671 SrcTy.getSizeInBits() == 64) &&
672 "Unexpected source type for SITOFP in X87 mode.");
673
674 TypeSize MemSize = SrcTy.getSizeInBytes();
675 MachinePointerInfo PtrInfo;
676 Align Alignmt = Helper.getStackTemporaryAlignment(Type: SrcTy);
677 auto SlotPointer = Helper.createStackTemporary(Bytes: MemSize, Alignment: Alignmt, PtrInfo);
678 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
679 PtrInfo, F: MachineMemOperand::MOStore, Size: MemSize, BaseAlignment: Align(MemSize));
680
681 // Store the integer value on the FPU stack.
682 MIRBuilder.buildStore(Val: Src, Addr: SlotPointer, MMO&: *StoreMMO);
683
684 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
685 PtrInfo, F: MachineMemOperand::MOLoad, Size: MemSize, BaseAlignment: Align(MemSize));
686 MIRBuilder.buildInstr(Opcode: X86::G_FILD)
687 .addDef(RegNo: Dst)
688 .addUse(RegNo: SlotPointer.getReg(Idx: 0))
689 .addMemOperand(MMO: LoadMMO);
690
691 MI.eraseFromParent();
692 return true;
693}
694
695bool X86LegalizerInfo::legalizeFPTOSI(MachineInstr &MI,
696 MachineRegisterInfo &MRI,
697 LegalizerHelper &Helper) const {
698 MachineFunction &MF = *MI.getMF();
699 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
700 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
701
702 TypeSize MemSize = DstTy.getSizeInBytes();
703 MachinePointerInfo PtrInfo;
704 Align Alignmt = Helper.getStackTemporaryAlignment(Type: DstTy);
705 auto SlotPointer = Helper.createStackTemporary(Bytes: MemSize, Alignment: Alignmt, PtrInfo);
706 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
707 PtrInfo, F: MachineMemOperand::MOStore, Size: MemSize, BaseAlignment: Align(MemSize));
708
709 MIRBuilder.buildInstr(Opcode: X86::G_FIST)
710 .addUse(RegNo: Src)
711 .addUse(RegNo: SlotPointer.getReg(Idx: 0))
712 .addMemOperand(MMO: StoreMMO);
713
714 MIRBuilder.buildLoad(Res: Dst, Addr: SlotPointer, PtrInfo, Alignment: Align(MemSize));
715 MI.eraseFromParent();
716 return true;
717}
718
719bool X86LegalizerInfo::legalizeBuildVector(MachineInstr &MI,
720 MachineRegisterInfo &MRI,
721 LegalizerHelper &Helper) const {
722 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
723 const auto &BuildVector = cast<GBuildVector>(Val&: MI);
724 Register Dst = BuildVector.getReg(Idx: 0);
725 LLT DstTy = MRI.getType(Reg: Dst);
726 MachineFunction &MF = MIRBuilder.getMF();
727 LLVMContext &Ctx = MF.getFunction().getContext();
728 uint64_t DstTySize = DstTy.getScalarSizeInBits();
729
730 SmallVector<Constant *, 4> CstIdxs;
731 for (unsigned i = 0; i < BuildVector.getNumSources(); ++i) {
732 Register Source = BuildVector.getSourceReg(I: i);
733
734 auto ValueAndReg = getIConstantVRegValWithLookThrough(VReg: Source, MRI);
735 if (ValueAndReg) {
736 CstIdxs.emplace_back(Args: ConstantInt::get(Context&: Ctx, V: ValueAndReg->Value));
737 continue;
738 }
739
740 auto FPValueAndReg = getFConstantVRegValWithLookThrough(VReg: Source, MRI);
741 if (FPValueAndReg) {
742 CstIdxs.emplace_back(Args: ConstantFP::get(Context&: Ctx, V: FPValueAndReg->Value));
743 continue;
744 }
745
746 if (getOpcodeDef<GImplicitDef>(Reg: Source, MRI)) {
747 CstIdxs.emplace_back(Args: UndefValue::get(T: Type::getIntNTy(C&: Ctx, N: DstTySize)));
748 continue;
749 }
750 return false;
751 }
752
753 Constant *ConstVal = ConstantVector::get(V: CstIdxs);
754
755 const DataLayout &DL = MIRBuilder.getDataLayout();
756 unsigned AddrSpace = DL.getDefaultGlobalsAddressSpace();
757 Align Alignment(DL.getABITypeAlign(Ty: ConstVal->getType()));
758 auto Addr = MIRBuilder.buildConstantPool(
759 Res: LLT::pointer(AddressSpace: AddrSpace, SizeInBits: DL.getPointerSizeInBits(AS: AddrSpace)),
760 Idx: MF.getConstantPool()->getConstantPoolIndex(C: ConstVal, Alignment));
761 MachineMemOperand *MMO =
762 MF.getMachineMemOperand(PtrInfo: MachinePointerInfo::getConstantPool(MF),
763 F: MachineMemOperand::MOLoad, MemTy: DstTy, BaseAlignment: Alignment);
764
765 MIRBuilder.buildLoad(Res: Dst, Addr, MMO&: *MMO);
766 MI.eraseFromParent();
767 return true;
768}
769
770bool X86LegalizerInfo::legalizeFPTOUI(MachineInstr &MI,
771 MachineRegisterInfo &MRI,
772 LegalizerHelper &Helper) const {
773 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
774 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
775 unsigned DstSizeInBits = DstTy.getScalarSizeInBits();
776 const LLT s32 = LLT::scalar(SizeInBits: 32);
777 const LLT s64 = LLT::scalar(SizeInBits: 64);
778
779 // Simply reuse FPTOSI when it is possible to widen the type
780 if (DstSizeInBits <= 32) {
781 auto Casted = MIRBuilder.buildFPTOSI(Dst: DstTy == s32 ? s64 : s32, Src0: Src);
782 MIRBuilder.buildTrunc(Res: Dst, Op: Casted);
783 MI.eraseFromParent();
784 return true;
785 }
786
787 return false;
788}
789
790bool X86LegalizerInfo::legalizeUITOFP(MachineInstr &MI,
791 MachineRegisterInfo &MRI,
792 LegalizerHelper &Helper) const {
793 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
794 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
795 const LLT s32 = LLT::scalar(SizeInBits: 32);
796 const LLT s64 = LLT::scalar(SizeInBits: 64);
797
798 // Simply reuse SITOFP when it is possible to widen the type
799 if (SrcTy.getSizeInBits() <= 32) {
800 auto Ext = MIRBuilder.buildZExt(Res: SrcTy == s32 ? s64 : s32, Op: Src);
801 MIRBuilder.buildSITOFP(Dst, Src0: Ext);
802 MI.eraseFromParent();
803 return true;
804 }
805
806 return false;
807}
808
809bool X86LegalizerInfo::legalizeNarrowingStore(MachineInstr &MI,
810 MachineRegisterInfo &MRI,
811 LegalizerHelper &Helper) const {
812 auto &Store = cast<GStore>(Val&: MI);
813 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
814 MachineMemOperand &MMO = **Store.memoperands_begin();
815 MachineFunction &MF = MIRBuilder.getMF();
816 LLT ValTy = MRI.getType(Reg: Store.getValueReg());
817 auto *NewMMO = MF.getMachineMemOperand(MMO: &MMO, PtrInfo: MMO.getPointerInfo(), Ty: ValTy);
818
819 Helper.Observer.changingInstr(MI&: Store);
820 Store.setMemRefs(MF, MemRefs: {NewMMO});
821 Helper.Observer.changedInstr(MI&: Store);
822 return true;
823}
824
825bool X86LegalizerInfo::legalizeGETROUNDING(MachineInstr &MI,
826 MachineRegisterInfo &MRI,
827 LegalizerHelper &Helper) const {
828 /*
829 The rounding mode is in bits 11:10 of FPSR, and has the following
830 settings:
831 00 Round to nearest
832 01 Round to -inf
833 10 Round to +inf
834 11 Round to 0
835
836 GET_ROUNDING, on the other hand, expects the following:
837 -1 Undefined
838 0 Round to 0
839 1 Round to nearest
840 2 Round to +inf
841 3 Round to -inf
842
843 To perform the conversion, we use a packed lookup table of the four 2-bit
844 values that we can index by FPSP[11:10]
845 0x2d --> (0b00,10,11,01) --> (0,2,3,1) >> FPSR[11:10]
846
847 (0x2d >> ((FPSR >> 9) & 6)) & 3
848 */
849
850 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
851 MachineFunction &MF = MIRBuilder.getMF();
852 Register Dst = MI.getOperand(i: 0).getReg();
853 LLT DstTy = MRI.getType(Reg: Dst);
854 const LLT s8 = LLT::scalar(SizeInBits: 8);
855 const LLT s16 = LLT::scalar(SizeInBits: 16);
856 const LLT s32 = LLT::scalar(SizeInBits: 32);
857
858 // Save FP Control Word to stack slot
859 int MemSize = 2;
860 Align Alignment = Align(2);
861 MachinePointerInfo PtrInfo;
862 auto StackTemp = Helper.createStackTemporary(Bytes: TypeSize::getFixed(ExactSize: MemSize),
863 Alignment, PtrInfo);
864 Register StackPtr = StackTemp.getReg(Idx: 0);
865
866 auto StoreMMO = MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOStore,
867 Size: MemSize, BaseAlignment: Alignment);
868
869 // Store FP Control Word to stack slot using G_FNSTCW16
870 MIRBuilder.buildInstr(Opcode: X86::G_FNSTCW16)
871 .addUse(RegNo: StackPtr)
872 .addMemOperand(MMO: StoreMMO);
873
874 // Load FP Control Word from stack slot
875 auto LoadMMO = MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOLoad,
876 Size: MemSize, BaseAlignment: Alignment);
877
878 auto CWD32 =
879 MIRBuilder.buildZExt(Res: s32, Op: MIRBuilder.buildLoad(Res: s16, Addr: StackPtr, MMO&: *LoadMMO));
880 auto Shifted8 = MIRBuilder.buildTrunc(
881 Res: s8, Op: MIRBuilder.buildLShr(Dst: s32, Src0: CWD32, Src1: MIRBuilder.buildConstant(Res: s8, Val: 9)));
882 auto Masked32 = MIRBuilder.buildZExt(
883 Res: s32, Op: MIRBuilder.buildAnd(Dst: s8, Src0: Shifted8, Src1: MIRBuilder.buildConstant(Res: s8, Val: 6)));
884
885 // LUT is a packed lookup table (0x2d) used to map the 2-bit x87 FPU rounding
886 // mode (from bits 11:10 of the control word) to the values expected by
887 // GET_ROUNDING. The mapping is performed by shifting LUT right by the
888 // extracted rounding mode and masking the result with 3 to obtain the final
889 auto LUT = MIRBuilder.buildConstant(Res: s32, Val: 0x2d);
890 auto LUTShifted = MIRBuilder.buildLShr(Dst: s32, Src0: LUT, Src1: Masked32);
891 auto RetVal =
892 MIRBuilder.buildAnd(Dst: s32, Src0: LUTShifted, Src1: MIRBuilder.buildConstant(Res: s32, Val: 3));
893 auto RetValTrunc = MIRBuilder.buildZExtOrTrunc(Res: DstTy, Op: RetVal);
894
895 MIRBuilder.buildCopy(Res: Dst, Op: RetValTrunc);
896
897 MI.eraseFromParent();
898 return true;
899}
900
901bool X86LegalizerInfo::legalizeSETROUNDING(MachineInstr &MI,
902 MachineRegisterInfo &MRI,
903 LegalizerHelper &Helper) const {
904 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
905 MachineFunction &MF = MIRBuilder.getMF();
906 Register Src = MI.getOperand(i: 0).getReg();
907 const LLT s8 = LLT::scalar(SizeInBits: 8);
908 const LLT s16 = LLT::scalar(SizeInBits: 16);
909 const LLT s32 = LLT::scalar(SizeInBits: 32);
910
911 // Allocate stack slot for control word and MXCSR (4 bytes).
912 int MemSize = 4;
913 Align Alignment = Align(4);
914 MachinePointerInfo PtrInfo;
915 auto StackTemp = Helper.createStackTemporary(Bytes: TypeSize::getFixed(ExactSize: MemSize),
916 Alignment, PtrInfo);
917 Register StackPtr = StackTemp.getReg(Idx: 0);
918
919 auto StoreMMO =
920 MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOStore, Size: 2, BaseAlignment: Align(2));
921 MIRBuilder.buildInstr(Opcode: X86::G_FNSTCW16)
922 .addUse(RegNo: StackPtr)
923 .addMemOperand(MMO: StoreMMO);
924
925 auto LoadMMO =
926 MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOLoad, Size: 2, BaseAlignment: Align(2));
927 auto CWD16 = MIRBuilder.buildLoad(Res: s16, Addr: StackPtr, MMO&: *LoadMMO);
928
929 // Clear RM field (bits 11:10)
930 auto ClearedCWD =
931 MIRBuilder.buildAnd(Dst: s16, Src0: CWD16, Src1: MIRBuilder.buildConstant(Res: s16, Val: 0xf3ff));
932
933 // Check if Src is a constant
934 Register RMBits;
935 Register MXCSRRMBits;
936
937 APInt SrcCst;
938 if (mi_match(R: Src, MRI, P: m_ICst(Cst&: SrcCst))) {
939 uint64_t RM = SrcCst.getZExtValue();
940 int FieldVal = X86::getRoundingModeX86(RM);
941
942 if (FieldVal == X86::rmInvalid) {
943 FieldVal = X86::rmToNearest;
944 LLVMContext &C = MF.getFunction().getContext();
945 C.diagnose(DI: DiagnosticInfoUnsupported(
946 MF.getFunction(), "rounding mode is not supported by X86 hardware",
947 DiagnosticLocation(MI.getDebugLoc()), DS_Error));
948 return false;
949 }
950
951 FieldVal = FieldVal << 3;
952 RMBits = MIRBuilder.buildConstant(Res: s16, Val: FieldVal).getReg(Idx: 0);
953 MXCSRRMBits = MIRBuilder.buildConstant(Res: s32, Val: FieldVal).getReg(Idx: 0);
954 } else {
955 // Convert Src (rounding mode) to bits for control word
956 // (0xc9 << (2 * Src + 4)) & 0xc00
957 auto Src32 = MIRBuilder.buildZExtOrTrunc(Res: s32, Op: Src);
958 auto ShiftAmt = MIRBuilder.buildAdd(
959 Dst: s32, Src0: MIRBuilder.buildShl(Dst: s32, Src0: Src32, Src1: MIRBuilder.buildConstant(Res: s32, Val: 1)),
960 Src1: MIRBuilder.buildConstant(Res: s32, Val: 4));
961 auto ShiftAmt8 = MIRBuilder.buildTrunc(Res: s8, Op: ShiftAmt);
962 auto Shifted = MIRBuilder.buildShl(Dst: s16, Src0: MIRBuilder.buildConstant(Res: s16, Val: 0xc9),
963 Src1: ShiftAmt8);
964 RMBits =
965 MIRBuilder.buildAnd(Dst: s16, Src0: Shifted, Src1: MIRBuilder.buildConstant(Res: s16, Val: 0xc00))
966 .getReg(Idx: 0);
967
968 // For non-constant case, we still need to compute MXCSR bits dynamically
969 auto RMBits32 = MIRBuilder.buildZExt(Res: s32, Op: RMBits);
970 MXCSRRMBits =
971 MIRBuilder.buildShl(Dst: s32, Src0: RMBits32, Src1: MIRBuilder.buildConstant(Res: s32, Val: 3))
972 .getReg(Idx: 0);
973 }
974 // Update rounding mode bits
975 auto NewCWD =
976 MIRBuilder.buildOr(Dst: s16, Src0: ClearedCWD, Src1: RMBits, Flags: MachineInstr::Disjoint);
977
978 // Store new FP Control Word to stack
979 auto StoreNewMMO =
980 MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOStore, Size: 2, BaseAlignment: Align(2));
981 MIRBuilder.buildStore(Val: NewCWD, Addr: StackPtr, MMO&: *StoreNewMMO);
982
983 // Load FP control word from the slot using G_FLDCW16
984 auto LoadNewMMO =
985 MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOLoad, Size: 2, BaseAlignment: Align(2));
986 MIRBuilder.buildInstr(Opcode: X86::G_FLDCW16)
987 .addUse(RegNo: StackPtr)
988 .addMemOperand(MMO: LoadNewMMO);
989
990 if (Subtarget.hasSSE1()) {
991 // Store MXCSR to stack (use STMXCSR)
992 auto StoreMXCSRMMO = MF.getMachineMemOperand(
993 PtrInfo, F: MachineMemOperand::MOStore, Size: 4, BaseAlignment: Align(4));
994 MIRBuilder.buildInstr(Opcode: TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS)
995 .addIntrinsicID(ID: Intrinsic::x86_sse_stmxcsr)
996 .addUse(RegNo: StackPtr)
997 .addMemOperand(MMO: StoreMXCSRMMO);
998
999 // Load MXCSR from stack
1000 auto LoadMXCSRMMO = MF.getMachineMemOperand(
1001 PtrInfo, F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: Align(4));
1002 auto MXCSR = MIRBuilder.buildLoad(Res: s32, Addr: StackPtr, MMO&: *LoadMXCSRMMO);
1003
1004 // Clear RM field (bits 14:13)
1005 auto ClearedMXCSR = MIRBuilder.buildAnd(
1006 Dst: s32, Src0: MXCSR, Src1: MIRBuilder.buildConstant(Res: s32, Val: 0xffff9fff));
1007
1008 // Update rounding mode bits
1009 auto NewMXCSR = MIRBuilder.buildOr(Dst: s32, Src0: ClearedMXCSR, Src1: MXCSRRMBits);
1010
1011 // Store new MXCSR to stack
1012 auto StoreNewMXCSRMMO = MF.getMachineMemOperand(
1013 PtrInfo, F: MachineMemOperand::MOStore, Size: 4, BaseAlignment: Align(4));
1014 MIRBuilder.buildStore(Val: NewMXCSR, Addr: StackPtr, MMO&: *StoreNewMXCSRMMO);
1015
1016 // Load MXCSR from stack (use LDMXCSR)
1017 auto LoadNewMXCSRMMO = MF.getMachineMemOperand(
1018 PtrInfo, F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: Align(4));
1019 MIRBuilder.buildInstr(Opcode: TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS)
1020 .addIntrinsicID(ID: Intrinsic::x86_sse_ldmxcsr)
1021 .addUse(RegNo: StackPtr)
1022 .addMemOperand(MMO: LoadNewMXCSRMMO);
1023 }
1024
1025 MI.eraseFromParent();
1026 return true;
1027}
1028
1029bool X86LegalizerInfo::legalizeGLOBAL_VALUE(MachineInstr &MI,
1030 MachineRegisterInfo &MRI,
1031 LegalizerHelper &Helper) const {
1032 const GlobalValue *GV = MI.getOperand(i: 1).getGlobal();
1033 Register Dst = MI.getOperand(i: 0).getReg();
1034 LLT DstTy = MRI.getType(Reg: Dst);
1035 unsigned GVOpFlags = Subtarget.classifyGlobalReference(GV);
1036
1037 // For stub references (GOT/PLT), we need G_WRAPPER_RIP + load
1038 if (isGlobalStubReference(TargetFlag: GVOpFlags)) {
1039 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1040 MachineFunction &MF = MIRBuilder.getMF();
1041
1042 Register StubAddr = MRI.createGenericVirtualRegister(Ty: DstTy);
1043 MIRBuilder.buildInstr(Opcode: X86::G_WRAPPER_RIP)
1044 .addDef(RegNo: StubAddr)
1045 .addGlobalAddress(GV);
1046
1047 MachineMemOperand *MMO = MF.getMachineMemOperand(
1048 PtrInfo: MachinePointerInfo::getGOT(MF), F: MachineMemOperand::MOLoad, MemTy: DstTy,
1049 BaseAlignment: Align(DstTy.getSizeInBytes()));
1050 MIRBuilder.buildLoad(Res: Dst, Addr: StubAddr, MMO&: *MMO);
1051 MI.eraseFromParent();
1052 }
1053 return true;
1054}
1055
1056bool X86LegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
1057 MachineInstr &MI) const {
1058 return true;
1059}
1060