1//===- X86LegalizerInfo.cpp --------------------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the Machinelegalizer class for X86.
10/// \todo This should be generated by TableGen.
11//===----------------------------------------------------------------------===//
12
13#include "X86LegalizerInfo.h"
14#include "X86Subtarget.h"
15#include "X86TargetMachine.h"
16#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
17#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
18#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
19#include "llvm/CodeGen/MachineConstantPool.h"
20#include "llvm/CodeGen/MachineFrameInfo.h"
21#include "llvm/CodeGen/TargetOpcodes.h"
22#include "llvm/CodeGen/ValueTypes.h"
23#include "llvm/IR/DerivedTypes.h"
24#include "llvm/IR/IntrinsicsX86.h"
25#include "llvm/IR/Type.h"
26
27using namespace llvm;
28using namespace TargetOpcode;
29using namespace LegalizeActions;
30using namespace LegalityPredicates;
31
32X86LegalizerInfo::X86LegalizerInfo(const X86Subtarget &STI,
33 const X86TargetMachine &TM)
34 : Subtarget(STI) {
35
36 bool Is64Bit = Subtarget.is64Bit();
37 bool HasCMOV = Subtarget.canUseCMOV();
38 bool HasSSE1 = Subtarget.hasSSE1();
39 bool HasSSE2 = Subtarget.hasSSE2();
40 bool HasSSE41 = Subtarget.hasSSE41();
41 bool HasAVX = Subtarget.hasAVX();
42 bool HasAVX2 = Subtarget.hasAVX2();
43 bool HasAVX512 = Subtarget.hasAVX512();
44 bool HasVLX = Subtarget.hasVLX();
45 bool HasDQI = Subtarget.hasAVX512() && Subtarget.hasDQI();
46 bool HasBWI = Subtarget.hasAVX512() && Subtarget.hasBWI();
47 bool UseX87 = !Subtarget.useSoftFloat() && Subtarget.hasX87();
48 bool HasPOPCNT = Subtarget.hasPOPCNT();
49 bool HasLZCNT = Subtarget.hasLZCNT();
50 bool HasBMI = Subtarget.hasBMI();
51
52 const LLT p0 = LLT::pointer(AddressSpace: 0, SizeInBits: TM.getPointerSizeInBits(AS: 0));
53 const LLT s1 = LLT::scalar(SizeInBits: 1);
54 const LLT s8 = LLT::scalar(SizeInBits: 8);
55 const LLT s16 = LLT::scalar(SizeInBits: 16);
56 const LLT s32 = LLT::scalar(SizeInBits: 32);
57 const LLT s64 = LLT::scalar(SizeInBits: 64);
58 const LLT s80 = LLT::scalar(SizeInBits: 80);
59 const LLT s128 = LLT::scalar(SizeInBits: 128);
60 const LLT sMaxScalar = Subtarget.is64Bit() ? s64 : s32;
61 const LLT v2s32 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 32);
62 const LLT v4s8 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 8);
63
64 const LLT v16s8 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 8);
65 const LLT v8s16 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 16);
66 const LLT v4s32 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 32);
67 const LLT v2s64 = LLT::fixed_vector(NumElements: 2, ScalarSizeInBits: 64);
68 const LLT v2p0 = LLT::fixed_vector(NumElements: 2, ScalarTy: p0);
69
70 const LLT v32s8 = LLT::fixed_vector(NumElements: 32, ScalarSizeInBits: 8);
71 const LLT v16s16 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 16);
72 const LLT v8s32 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 32);
73 const LLT v4s64 = LLT::fixed_vector(NumElements: 4, ScalarSizeInBits: 64);
74 const LLT v4p0 = LLT::fixed_vector(NumElements: 4, ScalarTy: p0);
75
76 const LLT v64s8 = LLT::fixed_vector(NumElements: 64, ScalarSizeInBits: 8);
77 const LLT v32s16 = LLT::fixed_vector(NumElements: 32, ScalarSizeInBits: 16);
78 const LLT v16s32 = LLT::fixed_vector(NumElements: 16, ScalarSizeInBits: 32);
79 const LLT v8s64 = LLT::fixed_vector(NumElements: 8, ScalarSizeInBits: 64);
80
81 const LLT s8MaxVector = HasAVX512 ? v64s8 : HasAVX ? v32s8 : v16s8;
82 const LLT s16MaxVector = HasAVX512 ? v32s16 : HasAVX ? v16s16 : v8s16;
83 const LLT s32MaxVector = HasAVX512 ? v16s32 : HasAVX ? v8s32 : v4s32;
84 const LLT s64MaxVector = HasAVX512 ? v8s64 : HasAVX ? v4s64 : v2s64;
85
86 // todo: AVX512 bool vector predicate types
87
88 // implicit/constants
89 // 32/64-bits needs support for s64/s128 to handle cases:
90 // s64 = EXTEND (G_IMPLICIT_DEF s32) -> s64 = G_IMPLICIT_DEF
91 // s128 = EXTEND (G_IMPLICIT_DEF s32/s64) -> s128 = G_IMPLICIT_DEF
92 getActionDefinitionsBuilder(
93 Opcodes: {G_IMPLICIT_DEF, G_PHI, G_FREEZE, G_CONSTANT_FOLD_BARRIER})
94 .legalFor(Types: {p0, s1, s8, s16, s32, s64})
95 .legalFor(Pred: UseX87, Types: {s80})
96 .legalFor(Pred: Is64Bit, Types: {s128})
97 .legalFor(Pred: HasSSE2, Types: {v16s8, v8s16, v4s32, v2s64})
98 .legalFor(Pred: HasAVX, Types: {v32s8, v16s16, v8s32, v4s64})
99 .legalFor(Pred: HasAVX512, Types: {v64s8, v32s16, v16s32, v8s64})
100 .widenScalarOrEltToNextPow2(TypeIdx: 0, /*Min=*/MinSize: 8)
101 .clampScalarOrElt(TypeIdx: 0, MinTy: s8, MaxTy: sMaxScalar)
102 .moreElementsToNextPow2(TypeIdx: 0)
103 .clampNumElements(TypeIdx: 0, MinTy: v16s8, MaxTy: s8MaxVector)
104 .clampNumElements(TypeIdx: 0, MinTy: v8s16, MaxTy: s16MaxVector)
105 .clampNumElements(TypeIdx: 0, MinTy: v4s32, MaxTy: s32MaxVector)
106 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: s64MaxVector)
107 .clampMaxNumElements(TypeIdx: 0, EltTy: p0,
108 MaxElements: Is64Bit ? s64MaxVector.getNumElements()
109 : s32MaxVector.getNumElements())
110 .scalarizeIf(Predicate: scalarOrEltWiderThan(TypeIdx: 0, Size: 64), TypeIdx: 0);
111
112 getActionDefinitionsBuilder(Opcode: G_CONSTANT)
113 .legalFor(Types: {p0, s8, s16, s32})
114 .legalFor(Pred: Is64Bit, Types: {s64})
115 .widenScalarToNextPow2(TypeIdx: 0, /*Min=*/MinSize: 8)
116 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: sMaxScalar);
117
118 getActionDefinitionsBuilder(Opcodes: {G_LROUND, G_LLROUND})
119 .widenScalarIf(Predicate: typeIs(TypeIdx: 1, TypesInit: s16),
120 Mutation: [=](const LegalityQuery &) {
121 return std::pair<unsigned, LLT>(1, s32);
122 })
123 .libcall();
124
125 getActionDefinitionsBuilder(
126 Opcodes: {G_FCOS, G_FCOSH, G_FACOS, G_FSIN, G_FSINH, G_FASIN, G_FTAN,
127 G_FTANH, G_FATAN, G_FATAN2, G_FPOW, G_FEXP, G_FEXP2, G_FEXP10,
128 G_FLOG, G_FLOG2, G_FLOG10, G_FPOWI, G_FSINCOS, G_FCEIL, G_FFLOOR})
129 .libcall();
130
131 getActionDefinitionsBuilder(Opcode: G_FNEG)
132 .legalFor(Pred: UseX87 && !HasSSE1, Types: {s32})
133 .legalFor(Pred: UseX87 && !HasSSE2, Types: {s64})
134 .legalFor(Pred: UseX87, Types: {s80})
135 .lower();
136
137 getActionDefinitionsBuilder(Opcode: G_FSQRT)
138 .legalFor(Pred: HasSSE1 || UseX87, Types: {s32})
139 .legalFor(Pred: HasSSE2 || UseX87, Types: {s64})
140 .legalFor(Pred: UseX87, Types: {s80});
141
142 getActionDefinitionsBuilder(Opcodes: {G_GET_ROUNDING, G_SET_ROUNDING})
143 .customFor(Types: {s32});
144
145 // merge/unmerge
146 for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
147 unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;
148 unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;
149 getActionDefinitionsBuilder(Opcode: Op)
150 .widenScalarToNextPow2(TypeIdx: LitTyIdx, /*Min=*/MinSize: 8)
151 .widenScalarToNextPow2(TypeIdx: BigTyIdx, /*Min=*/MinSize: 16)
152 .minScalar(TypeIdx: LitTyIdx, Ty: s8)
153 .minScalar(TypeIdx: BigTyIdx, Ty: s32)
154 .legalIf(Predicate: [=](const LegalityQuery &Q) {
155 switch (Q.Types[BigTyIdx].getSizeInBits()) {
156 case 16:
157 case 32:
158 case 64:
159 case 128:
160 case 256:
161 case 512:
162 break;
163 default:
164 return false;
165 }
166 switch (Q.Types[LitTyIdx].getSizeInBits()) {
167 case 8:
168 case 16:
169 case 32:
170 case 64:
171 case 128:
172 case 256:
173 return true;
174 default:
175 return false;
176 }
177 });
178 }
179
180 getActionDefinitionsBuilder(Opcodes: {G_UMIN, G_UMAX, G_SMIN, G_SMAX})
181 .widenScalarToNextPow2(TypeIdx: 0, /*Min=*/MinSize: 32)
182 .lower();
183
184 // integer addition/subtraction
185 getActionDefinitionsBuilder(Opcodes: {G_ADD, G_SUB})
186 .legalFor(Types: {s8, s16, s32})
187 .legalFor(Pred: Is64Bit, Types: {s64})
188 .legalFor(Pred: HasSSE2, Types: {v16s8, v8s16, v4s32, v2s64})
189 .legalFor(Pred: HasAVX2, Types: {v32s8, v16s16, v8s32, v4s64})
190 .legalFor(Pred: HasAVX512, Types: {v16s32, v8s64})
191 .legalFor(Pred: HasBWI, Types: {v64s8, v32s16})
192 .clampMinNumElements(TypeIdx: 0, EltTy: s8, MinElements: 16)
193 .clampMinNumElements(TypeIdx: 0, EltTy: s16, MinElements: 8)
194 .clampMinNumElements(TypeIdx: 0, EltTy: s32, MinElements: 4)
195 .clampMinNumElements(TypeIdx: 0, EltTy: s64, MinElements: 2)
196 .clampMaxNumElements(TypeIdx: 0, EltTy: s8, MaxElements: HasBWI ? 64 : (HasAVX2 ? 32 : 16))
197 .clampMaxNumElements(TypeIdx: 0, EltTy: s16, MaxElements: HasBWI ? 32 : (HasAVX2 ? 16 : 8))
198 .clampMaxNumElements(TypeIdx: 0, EltTy: s32, MaxElements: HasAVX512 ? 16 : (HasAVX2 ? 8 : 4))
199 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: HasAVX512 ? 8 : (HasAVX2 ? 4 : 2))
200 .widenScalarToNextPow2(TypeIdx: 0, /*Min=*/MinSize: 32)
201 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: sMaxScalar)
202 .scalarize(TypeIdx: 0);
203
204 getActionDefinitionsBuilder(Opcodes: {G_UADDE, G_UADDO, G_USUBE, G_USUBO})
205 .legalFor(Types: {{s8, s8}, {s16, s8}, {s32, s8}})
206 .legalFor(Pred: Is64Bit, Types: {{s64, s8}})
207 .widenScalarToNextPow2(TypeIdx: 0, /*Min=*/MinSize: 32)
208 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: sMaxScalar)
209 .clampScalar(TypeIdx: 1, MinTy: s8, MaxTy: s8)
210 .scalarize(TypeIdx: 0);
211
212 // integer multiply
213 getActionDefinitionsBuilder(Opcode: G_MUL)
214 .legalFor(Types: {s8, s16, s32})
215 .legalFor(Pred: Is64Bit, Types: {s64})
216 .legalFor(Pred: HasSSE2, Types: {v8s16})
217 .legalFor(Pred: HasSSE41, Types: {v4s32})
218 .legalFor(Pred: HasAVX2, Types: {v16s16, v8s32})
219 .legalFor(Pred: HasAVX512, Types: {v16s32})
220 .legalFor(Pred: HasDQI, Types: {v8s64})
221 .legalFor(Pred: HasDQI && HasVLX, Types: {v2s64, v4s64})
222 .legalFor(Pred: HasBWI, Types: {v32s16})
223 .clampMinNumElements(TypeIdx: 0, EltTy: s16, MinElements: 8)
224 .clampMinNumElements(TypeIdx: 0, EltTy: s32, MinElements: 4)
225 .clampMinNumElements(TypeIdx: 0, EltTy: s64, MinElements: HasVLX ? 2 : 8)
226 .clampMaxNumElements(TypeIdx: 0, EltTy: s16, MaxElements: HasBWI ? 32 : (HasAVX2 ? 16 : 8))
227 .clampMaxNumElements(TypeIdx: 0, EltTy: s32, MaxElements: HasAVX512 ? 16 : (HasAVX2 ? 8 : 4))
228 .clampMaxNumElements(TypeIdx: 0, EltTy: s64, MaxElements: 8)
229 .widenScalarToNextPow2(TypeIdx: 0, /*Min=*/MinSize: 32)
230 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: sMaxScalar)
231 .scalarize(TypeIdx: 0);
232
233 getActionDefinitionsBuilder(Opcodes: {G_SMULH, G_UMULH})
234 .legalFor(Types: {s8, s16, s32})
235 .legalFor(Pred: Is64Bit, Types: {s64})
236 .widenScalarToNextPow2(TypeIdx: 0, /*Min=*/MinSize: 32)
237 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: sMaxScalar)
238 .scalarize(TypeIdx: 0);
239
240 // integer divisions
241 getActionDefinitionsBuilder(Opcodes: {G_SDIV, G_SREM, G_UDIV, G_UREM})
242 .legalFor(Types: {s8, s16, s32})
243 .legalFor(Pred: Is64Bit, Types: {s64})
244 .libcallFor(Types: {s64})
245 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: sMaxScalar);
246
247 // integer shifts
248 getActionDefinitionsBuilder(Opcodes: {G_SHL, G_LSHR, G_ASHR})
249 .legalFor(Types: {{s8, s8}, {s16, s8}, {s32, s8}})
250 .legalFor(Pred: Is64Bit, Types: {{s64, s8}})
251 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: sMaxScalar)
252 .clampScalar(TypeIdx: 1, MinTy: s8, MaxTy: s8);
253
254 // integer logic
255 getActionDefinitionsBuilder(Opcodes: {G_AND, G_OR, G_XOR})
256 .legalFor(Types: {s8, s16, s32})
257 .legalFor(Pred: Is64Bit, Types: {s64})
258 .legalFor(Pred: HasSSE2, Types: {v16s8, v8s16, v4s32, v2s64})
259 .legalFor(Pred: HasAVX, Types: {v32s8, v16s16, v8s32, v4s64})
260 .legalFor(Pred: HasAVX512, Types: {v64s8, v32s16, v16s32, v8s64})
261 .clampNumElements(TypeIdx: 0, MinTy: v16s8, MaxTy: s8MaxVector)
262 .clampNumElements(TypeIdx: 0, MinTy: v8s16, MaxTy: s16MaxVector)
263 .clampNumElements(TypeIdx: 0, MinTy: v4s32, MaxTy: s32MaxVector)
264 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: s64MaxVector)
265 .widenScalarToNextPow2(TypeIdx: 0, /*Min=*/MinSize: 32)
266 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: sMaxScalar)
267 .scalarize(TypeIdx: 0);
268
269 // integer comparison
270 const std::initializer_list<LLT> IntTypes32 = {s8, s16, s32, p0};
271 const std::initializer_list<LLT> IntTypes64 = {s8, s16, s32, s64, p0};
272
273 getActionDefinitionsBuilder(Opcode: G_ICMP)
274 .legalForCartesianProduct(Types0: {s8}, Types1: Is64Bit ? IntTypes64 : IntTypes32)
275 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: s8)
276 .widenScalarToNextPow2(TypeIdx: 1, /*Min=*/MinSize: 8)
277 .clampScalar(TypeIdx: 1, MinTy: s8, MaxTy: sMaxScalar);
278
279 // bswap
280 getActionDefinitionsBuilder(Opcode: G_BSWAP)
281 .legalFor(Types: {s32})
282 .legalFor(Pred: Is64Bit, Types: {s64})
283 .widenScalarToNextPow2(TypeIdx: 0, /*Min=*/MinSize: 32)
284 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: sMaxScalar);
285
286 // popcount
287 getActionDefinitionsBuilder(Opcode: G_CTPOP)
288 .legalFor(Pred: HasPOPCNT, Types: {{s16, s16}, {s32, s32}})
289 .legalFor(Pred: HasPOPCNT && Is64Bit, Types: {{s64, s64}})
290 .widenScalarToNextPow2(TypeIdx: 1, /*Min=*/MinSize: 16)
291 .clampScalar(TypeIdx: 1, MinTy: s16, MaxTy: sMaxScalar)
292 .scalarSameSizeAs(TypeIdx: 0, SameSizeIdx: 1);
293
294 // count leading zeros (LZCNT)
295 getActionDefinitionsBuilder(Opcode: G_CTLZ)
296 .legalFor(Pred: HasLZCNT, Types: {{s16, s16}, {s32, s32}})
297 .legalFor(Pred: HasLZCNT && Is64Bit, Types: {{s64, s64}})
298 .widenScalarToNextPow2(TypeIdx: 1, /*Min=*/MinSize: 16)
299 .clampScalar(TypeIdx: 1, MinTy: s16, MaxTy: sMaxScalar)
300 .scalarSameSizeAs(TypeIdx: 0, SameSizeIdx: 1);
301
302 // count trailing zeros
303 getActionDefinitionsBuilder(Opcode: G_CTTZ_ZERO_POISON)
304 .legalFor(Types: {{s16, s16}, {s32, s32}})
305 .legalFor(Pred: Is64Bit, Types: {{s64, s64}})
306 .widenScalarToNextPow2(TypeIdx: 1, /*Min=*/MinSize: 16)
307 .clampScalar(TypeIdx: 1, MinTy: s16, MaxTy: sMaxScalar)
308 .scalarSameSizeAs(TypeIdx: 0, SameSizeIdx: 1);
309
310 getActionDefinitionsBuilder(Opcode: G_CTTZ)
311 .legalFor(Pred: HasBMI, Types: {{s16, s16}, {s32, s32}})
312 .legalFor(Pred: HasBMI && Is64Bit, Types: {{s64, s64}})
313 .widenScalarToNextPow2(TypeIdx: 1, /*Min=*/MinSize: 16)
314 .clampScalar(TypeIdx: 1, MinTy: s16, MaxTy: sMaxScalar)
315 .scalarSameSizeAs(TypeIdx: 0, SameSizeIdx: 1);
316
317 getActionDefinitionsBuilder(Opcode: G_BR).alwaysLegal();
318 getActionDefinitionsBuilder(Opcode: G_BRCOND).legalFor(Types: {s1});
319
320 // pointer handling
321 const std::initializer_list<LLT> PtrTypes32 = {s1, s8, s16, s32};
322 const std::initializer_list<LLT> PtrTypes64 = {s1, s8, s16, s32, s64};
323
324 getActionDefinitionsBuilder(Opcode: G_PTRTOINT)
325 .legalForCartesianProduct(Types0: Is64Bit ? PtrTypes64 : PtrTypes32, Types1: {p0})
326 .maxScalar(TypeIdx: 0, Ty: sMaxScalar)
327 .widenScalarToNextPow2(TypeIdx: 0, /*Min*/ MinSize: 8);
328
329 getActionDefinitionsBuilder(Opcode: G_INTTOPTR).legalFor(Types: {{p0, sMaxScalar}});
330
331 getActionDefinitionsBuilder(Opcode: G_CONSTANT_POOL).legalFor(Types: {p0});
332
333 getActionDefinitionsBuilder(Opcode: G_PTR_ADD)
334 .legalFor(Types: {{p0, s32}})
335 .legalFor(Pred: Is64Bit, Types: {{p0, s64}})
336 .widenScalarToNextPow2(TypeIdx: 1, /*Min*/ MinSize: 32)
337 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: sMaxScalar);
338
339 getActionDefinitionsBuilder(Opcode: G_FRAME_INDEX).legalFor(Types: {p0});
340
341 getActionDefinitionsBuilder(Opcode: G_GLOBAL_VALUE).customFor(Types: {p0});
342
343 // load/store: add more corner cases
344 for (unsigned Op : {G_LOAD, G_STORE}) {
345 auto &Action = getActionDefinitionsBuilder(Opcode: Op);
346 Action.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: s8, .Type1: p0, .MemTy: s8, .Align: 1},
347 {.Type0: s16, .Type1: p0, .MemTy: s16, .Align: 1},
348 {.Type0: s32, .Type1: p0, .MemTy: s32, .Align: 1},
349 {.Type0: s80, .Type1: p0, .MemTy: s80, .Align: 1},
350 {.Type0: p0, .Type1: p0, .MemTy: p0, .Align: 1},
351 {.Type0: v4s8, .Type1: p0, .MemTy: v4s8, .Align: 1}});
352 if (Is64Bit)
353 Action.legalForTypesWithMemDesc(
354 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s64, .Align: 1}, {.Type0: v2s32, .Type1: p0, .MemTy: v2s32, .Align: 1}});
355
356 if (HasSSE1)
357 Action.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: v4s32, .Type1: p0, .MemTy: v4s32, .Align: 1}});
358 if (HasSSE2)
359 Action.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: v16s8, .Type1: p0, .MemTy: v16s8, .Align: 1},
360 {.Type0: v8s16, .Type1: p0, .MemTy: v8s16, .Align: 1},
361 {.Type0: v2s64, .Type1: p0, .MemTy: v2s64, .Align: 1},
362 {.Type0: v2p0, .Type1: p0, .MemTy: v2p0, .Align: 1}});
363 if (HasAVX)
364 Action.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: v32s8, .Type1: p0, .MemTy: v32s8, .Align: 1},
365 {.Type0: v16s16, .Type1: p0, .MemTy: v16s16, .Align: 1},
366 {.Type0: v8s32, .Type1: p0, .MemTy: v8s32, .Align: 1},
367 {.Type0: v4s64, .Type1: p0, .MemTy: v4s64, .Align: 1},
368 {.Type0: v4p0, .Type1: p0, .MemTy: v4p0, .Align: 1}});
369 if (HasAVX512)
370 Action.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: v64s8, .Type1: p0, .MemTy: v64s8, .Align: 1},
371 {.Type0: v32s16, .Type1: p0, .MemTy: v32s16, .Align: 1},
372 {.Type0: v16s32, .Type1: p0, .MemTy: v16s32, .Align: 1},
373 {.Type0: v8s64, .Type1: p0, .MemTy: v8s64, .Align: 1}});
374
375 // X86 supports extending loads but not stores for GPRs
376 if (Op == G_LOAD) {
377 Action.legalForTypesWithMemDesc(TypesAndMemDesc: {{.Type0: s8, .Type1: p0, .MemTy: s1, .Align: 1},
378 {.Type0: s16, .Type1: p0, .MemTy: s8, .Align: 1},
379 {.Type0: s32, .Type1: p0, .MemTy: s8, .Align: 1},
380 {.Type0: s32, .Type1: p0, .MemTy: s16, .Align: 1}});
381 if (Is64Bit)
382 Action.legalForTypesWithMemDesc(
383 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s8, .Align: 1}, {.Type0: s64, .Type1: p0, .MemTy: s16, .Align: 1}, {.Type0: s64, .Type1: p0, .MemTy: s32, .Align: 1}});
384 } else {
385 Action.customIf(Predicate: [=](const LegalityQuery &Query) {
386 return Query.Types[0] != Query.MMODescrs[0].MemoryTy;
387 });
388 }
389 Action.widenScalarToNextPow2(TypeIdx: 0, /*Min=*/MinSize: 8)
390 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: sMaxScalar)
391 .scalarize(TypeIdx: 0);
392 }
393
394 for (unsigned Op : {G_SEXTLOAD, G_ZEXTLOAD}) {
395 auto &Action = getActionDefinitionsBuilder(Opcode: Op);
396 Action.legalForTypesWithMemDesc(
397 TypesAndMemDesc: {{.Type0: s16, .Type1: p0, .MemTy: s8, .Align: 1}, {.Type0: s32, .Type1: p0, .MemTy: s8, .Align: 1}, {.Type0: s32, .Type1: p0, .MemTy: s16, .Align: 1}});
398 if (Is64Bit)
399 Action.legalForTypesWithMemDesc(
400 TypesAndMemDesc: {{.Type0: s64, .Type1: p0, .MemTy: s8, .Align: 1}, {.Type0: s64, .Type1: p0, .MemTy: s16, .Align: 1}, {.Type0: s64, .Type1: p0, .MemTy: s32, .Align: 1}});
401 // TODO - SSE41/AVX2/AVX512F/AVX512BW vector extensions
402 }
403
404 for (unsigned Op : {G_FPEXTLOAD, G_FPTRUNCSTORE}) {
405 auto &Action = getActionDefinitionsBuilder(Opcode: Op);
406 Action.legalForTypesWithMemDesc(
407 Pred: UseX87, TypesAndMemDesc: {{.Type0: s80, .Type1: p0, .MemTy: s32, .Align: 1}, {.Type0: s80, .Type1: p0, .MemTy: s64, .Align: 1}, {.Type0: s64, .Type1: p0, .MemTy: s32, .Align: 1}});
408 }
409
410 // sext, zext, and anyext
411 getActionDefinitionsBuilder(Opcode: G_ANYEXT)
412 .legalFor(Types: {s8, s16, s32, s128})
413 .legalFor(Pred: Is64Bit, Types: {s64})
414 .widenScalarToNextPow2(TypeIdx: 0, /*Min=*/MinSize: 8)
415 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: sMaxScalar)
416 .widenScalarToNextPow2(TypeIdx: 1, /*Min=*/MinSize: 8)
417 .clampScalar(TypeIdx: 1, MinTy: s8, MaxTy: sMaxScalar)
418 .scalarize(TypeIdx: 0);
419
420 getActionDefinitionsBuilder(Opcodes: {G_SEXT, G_ZEXT})
421 .legalFor(Types: {s8, s16, s32})
422 .legalFor(Pred: Is64Bit, Types: {s64})
423 .widenScalarToNextPow2(TypeIdx: 0, /*Min=*/MinSize: 8)
424 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: sMaxScalar)
425 .widenScalarToNextPow2(TypeIdx: 1, /*Min=*/MinSize: 8)
426 .clampScalar(TypeIdx: 1, MinTy: s8, MaxTy: sMaxScalar)
427 .scalarize(TypeIdx: 0);
428
429 getActionDefinitionsBuilder(Opcode: G_TRUNC).legalForCartesianProduct(
430 Types0: {s1, s8, s16, s32, s64}, Types1: {s8, s16, s32, s64, s128});
431
432 getActionDefinitionsBuilder(Opcode: G_SEXT_INREG).lower();
433
434 // fp constants
435 getActionDefinitionsBuilder(Opcode: G_FCONSTANT)
436 .legalFor(Types: {s32, s64})
437 .legalFor(Pred: UseX87, Types: {s80});
438
439 // fp arithmetic
440 getActionDefinitionsBuilder(Opcodes: {G_FADD, G_FSUB, G_FMUL, G_FDIV})
441 .legalFor(Types: {s32, s64})
442 .legalFor(Pred: HasSSE1, Types: {v4s32})
443 .legalFor(Pred: HasSSE2, Types: {v2s64})
444 .legalFor(Pred: HasAVX, Types: {v8s32, v4s64})
445 .legalFor(Pred: HasAVX512, Types: {v16s32, v8s64})
446 .legalFor(Pred: UseX87, Types: {s80});
447
448 getActionDefinitionsBuilder(Opcode: G_FABS)
449 .legalFor(Pred: UseX87, Types: {s80})
450 .legalFor(Pred: UseX87 && !Is64Bit, Types: {s64})
451 .lower();
452
453 // fp comparison
454 getActionDefinitionsBuilder(Opcode: G_FCMP)
455 .legalFor(Pred: HasSSE1 || UseX87, Types: {s8, s32})
456 .legalFor(Pred: HasSSE2 || UseX87, Types: {s8, s64})
457 .legalFor(Pred: UseX87, Types: {s8, s80})
458 .clampScalar(TypeIdx: 0, MinTy: s8, MaxTy: s8)
459 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: HasSSE2 ? s64 : s32)
460 .widenScalarToNextPow2(TypeIdx: 1);
461
462 // fp conversions
463 getActionDefinitionsBuilder(Opcode: G_FPEXT)
464 .legalFor(Pred: HasSSE2, Types: {{s64, s32}})
465 .legalFor(Pred: HasAVX, Types: {{v4s64, v4s32}})
466 .legalFor(Pred: HasAVX512, Types: {{v8s64, v8s32}})
467 .lowerFor(Pred: UseX87, Types: {{s64, s32}, {s80, s32}, {s80, s64}})
468 .libcall();
469
470 getActionDefinitionsBuilder(Opcode: G_FPTRUNC)
471 .legalFor(Pred: HasSSE2, Types: {{s32, s64}})
472 .legalFor(Pred: HasAVX, Types: {{v4s32, v4s64}})
473 .legalFor(Pred: HasAVX512, Types: {{v8s32, v8s64}})
474 .lowerFor(Pred: UseX87, Types: {{s32, s64}, {s32, s80}, {s64, s80}});
475
476 getActionDefinitionsBuilder(Opcode: G_SITOFP)
477 .legalFor(Pred: HasSSE1, Types: {{s32, s32}})
478 .legalFor(Pred: HasSSE1 && Is64Bit, Types: {{s32, s64}})
479 .legalFor(Pred: HasSSE2, Types: {{s64, s32}})
480 .legalFor(Pred: HasSSE2 && Is64Bit, Types: {{s64, s64}})
481 .clampScalar(TypeIdx: 1, MinTy: (UseX87 && !HasSSE1) ? s16 : s32, MaxTy: sMaxScalar)
482 .widenScalarToNextPow2(TypeIdx: 1)
483 .customForCartesianProduct(Pred: UseX87, Types0: {s32, s64, s80}, Types1: {s16, s32, s64})
484 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: HasSSE2 ? s64 : s32)
485 .widenScalarToNextPow2(TypeIdx: 0);
486
487 getActionDefinitionsBuilder(Opcode: G_FPTOSI)
488 .legalFor(Pred: HasSSE1, Types: {{s32, s32}})
489 .legalFor(Pred: HasSSE1 && Is64Bit, Types: {{s64, s32}})
490 .legalFor(Pred: HasSSE2, Types: {{s32, s64}})
491 .legalFor(Pred: HasSSE2 && Is64Bit, Types: {{s64, s64}})
492 .clampScalar(TypeIdx: 0, MinTy: (UseX87 && !HasSSE1) ? s16 : s32, MaxTy: sMaxScalar)
493 .widenScalarToNextPow2(TypeIdx: 0)
494 .customForCartesianProduct(Pred: UseX87, Types0: {s16, s32, s64}, Types1: {s32, s64, s80})
495 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: HasSSE2 ? s64 : s32)
496 .widenScalarToNextPow2(TypeIdx: 1);
497
498 // For G_UITOFP and G_FPTOUI without AVX512, we have to custom legalize types
499 // <= s32 manually. Otherwise, in custom handler there is no way to
500 // understand whether s32 is an original type and we need to promote it to
501 // s64 or s32 is obtained after widening and we shouldn't widen it to s64.
502 //
503 // For AVX512 we simply widen types as there is direct mapping from opcodes
504 // to asm instructions.
505 getActionDefinitionsBuilder(Opcode: G_UITOFP)
506 .legalFor(Pred: HasAVX512, Types: {{s32, s32}, {s32, s64}, {s64, s32}, {s64, s64}})
507 .customIf(Predicate: [=](const LegalityQuery &Query) {
508 return !HasAVX512 &&
509 ((HasSSE1 && typeIs(TypeIdx: 0, TypesInit: s32)(Query)) ||
510 (HasSSE2 && typeIs(TypeIdx: 0, TypesInit: s64)(Query))) &&
511 scalarNarrowerThan(TypeIdx: 1, Size: Is64Bit ? 64 : 32)(Query);
512 })
513 .lowerIf(Predicate: [=](const LegalityQuery &Query) {
514 // Lower conversions from s64
515 return !HasAVX512 &&
516 ((HasSSE1 && typeIs(TypeIdx: 0, TypesInit: s32)(Query)) ||
517 (HasSSE2 && typeIs(TypeIdx: 0, TypesInit: s64)(Query))) &&
518 (Is64Bit && typeIs(TypeIdx: 1, TypesInit: s64)(Query));
519 })
520 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: HasSSE2 ? s64 : s32)
521 .widenScalarToNextPow2(TypeIdx: 0)
522 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: sMaxScalar)
523 .widenScalarToNextPow2(TypeIdx: 1);
524
525 getActionDefinitionsBuilder(Opcode: G_FPTOUI)
526 .legalFor(Pred: HasAVX512, Types: {{s32, s32}, {s32, s64}, {s64, s32}, {s64, s64}})
527 .customIf(Predicate: [=](const LegalityQuery &Query) {
528 return !HasAVX512 &&
529 ((HasSSE1 && typeIs(TypeIdx: 1, TypesInit: s32)(Query)) ||
530 (HasSSE2 && typeIs(TypeIdx: 1, TypesInit: s64)(Query))) &&
531 scalarNarrowerThan(TypeIdx: 0, Size: Is64Bit ? 64 : 32)(Query);
532 })
533 // TODO: replace with customized legalization using
534 // specifics of cvttsd2si. The selection of this node requires
535 // a vector type. Either G_SCALAR_TO_VECTOR is needed or more advanced
536 // support of G_BUILD_VECTOR/G_INSERT_VECTOR_ELT is required beforehand.
537 .lowerIf(Predicate: [=](const LegalityQuery &Query) {
538 return !HasAVX512 &&
539 ((HasSSE1 && typeIs(TypeIdx: 1, TypesInit: s32)(Query)) ||
540 (HasSSE2 && typeIs(TypeIdx: 1, TypesInit: s64)(Query))) &&
541 (Is64Bit && typeIs(TypeIdx: 0, TypesInit: s64)(Query));
542 })
543 .clampScalar(TypeIdx: 0, MinTy: s32, MaxTy: sMaxScalar)
544 .widenScalarToNextPow2(TypeIdx: 0)
545 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: HasSSE2 ? s64 : s32)
546 .widenScalarToNextPow2(TypeIdx: 1);
547
548 // vector ops
549 getActionDefinitionsBuilder(Opcode: G_BUILD_VECTOR)
550 .customIf(Predicate: [=](const LegalityQuery &Query) {
551 return (HasSSE1 && typeInSet(TypeIdx: 0, TypesInit: {v4s32})(Query)) ||
552 (HasSSE2 && typeInSet(TypeIdx: 0, TypesInit: {v2s64, v8s16, v16s8})(Query)) ||
553 (HasAVX && typeInSet(TypeIdx: 0, TypesInit: {v4s64, v8s32, v16s16, v32s8})(Query)) ||
554 (HasAVX512 &&
555 typeInSet(TypeIdx: 0, TypesInit: {v8s64, v16s32, v32s16, v64s8})(Query));
556 })
557 .clampNumElements(TypeIdx: 0, MinTy: v16s8, MaxTy: s8MaxVector)
558 .clampNumElements(TypeIdx: 0, MinTy: v8s16, MaxTy: s16MaxVector)
559 .clampNumElements(TypeIdx: 0, MinTy: v4s32, MaxTy: s32MaxVector)
560 .clampNumElements(TypeIdx: 0, MinTy: v2s64, MaxTy: s64MaxVector)
561 .moreElementsToNextPow2(TypeIdx: 0);
562
563 getActionDefinitionsBuilder(Opcodes: {G_EXTRACT, G_INSERT})
564 .legalIf(Predicate: [=](const LegalityQuery &Query) {
565 unsigned SubIdx = Query.Opcode == G_EXTRACT ? 0 : 1;
566 unsigned FullIdx = Query.Opcode == G_EXTRACT ? 1 : 0;
567 return (HasAVX && typePairInSet(TypeIdx0: SubIdx, TypeIdx1: FullIdx,
568 TypesInit: {{v16s8, v32s8},
569 {v8s16, v16s16},
570 {v4s32, v8s32},
571 {v2s64, v4s64}})(Query)) ||
572 (HasAVX512 && typePairInSet(TypeIdx0: SubIdx, TypeIdx1: FullIdx,
573 TypesInit: {{v16s8, v64s8},
574 {v32s8, v64s8},
575 {v8s16, v32s16},
576 {v16s16, v32s16},
577 {v4s32, v16s32},
578 {v8s32, v16s32},
579 {v2s64, v8s64},
580 {v4s64, v8s64}})(Query));
581 });
582
583 // todo: only permit dst types up to max legal vector register size?
584 getActionDefinitionsBuilder(Opcode: G_CONCAT_VECTORS)
585 .legalFor(
586 Pred: HasSSE1,
587 Types: {{v32s8, v16s8}, {v16s16, v8s16}, {v8s32, v4s32}, {v4s64, v2s64}})
588 .legalFor(Pred: HasAVX, Types: {{v64s8, v16s8},
589 {v64s8, v32s8},
590 {v32s16, v8s16},
591 {v32s16, v16s16},
592 {v16s32, v4s32},
593 {v16s32, v8s32},
594 {v8s64, v2s64},
595 {v8s64, v4s64}});
596
597 // todo: vectors and address spaces
598 getActionDefinitionsBuilder(Opcode: G_SELECT)
599 .legalFor(Types: {{s16, s32}, {s32, s32}, {p0, s32}})
600 .legalFor(Pred: !HasCMOV, Types: {{s8, s32}})
601 .legalFor(Pred: Is64Bit, Types: {{s64, s32}})
602 .legalFor(Pred: UseX87, Types: {{s80, s32}})
603 .clampScalar(TypeIdx: 1, MinTy: s32, MaxTy: s32)
604 .widenScalarToNextPow2(TypeIdx: 0, /*Min=*/MinSize: 8)
605 .clampScalar(TypeIdx: 0, MinTy: HasCMOV ? s16 : s8, MaxTy: sMaxScalar);
606
607 // memory intrinsics
608 getActionDefinitionsBuilder(Opcodes: {G_MEMCPY, G_MEMMOVE, G_MEMSET}).libcall();
609
610 getActionDefinitionsBuilder(Opcodes: {G_DYN_STACKALLOC, G_STACKSAVE, G_STACKRESTORE})
611 .lower();
612
613 // fp intrinsics
614 // fpclass for i686 is disabled for llvm issue #171992
615 getActionDefinitionsBuilder(Opcode: G_IS_FPCLASS)
616 .lowerFor(Pred: Is64Bit, Types: {{s1, s32}, {s1, s64}, {s1, s80}});
617
618 getActionDefinitionsBuilder(Opcodes: {G_INTRINSIC_ROUNDEVEN, G_INTRINSIC_TRUNC})
619 .scalarize(TypeIdx: 0)
620 .minScalar(TypeIdx: 0, Ty: LLT::scalar(SizeInBits: 32))
621 .libcall();
622
623 getActionDefinitionsBuilder(Opcodes: {G_INTRINSIC, G_INTRINSIC_W_SIDE_EFFECTS})
624 .alwaysLegal();
625 getActionDefinitionsBuilder(Opcodes: {G_TRAP, G_DEBUGTRAP, G_UBSANTRAP}).alwaysLegal();
626 getActionDefinitionsBuilder(Opcode: G_INVOKE_REGION_START).alwaysLegal();
627
628 verify(MII: *STI.getInstrInfo());
629}
630
631bool X86LegalizerInfo::legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI,
632 LostDebugLocObserver &LocObserver) const {
633 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
634 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
635 switch (MI.getOpcode()) {
636 default:
637 // No idea what to do.
638 return false;
639 case TargetOpcode::G_BUILD_VECTOR:
640 return legalizeBuildVector(MI, MRI, Helper);
641 case TargetOpcode::G_FPTOUI:
642 return legalizeFPTOUI(MI, MRI, Helper);
643 case TargetOpcode::G_UITOFP:
644 return legalizeUITOFP(MI, MRI, Helper);
645 case TargetOpcode::G_STORE:
646 return legalizeNarrowingStore(MI, MRI, Helper);
647 case TargetOpcode::G_SITOFP:
648 return legalizeSITOFP(MI, MRI, Helper);
649 case TargetOpcode::G_FPTOSI:
650 return legalizeFPTOSI(MI, MRI, Helper);
651 case TargetOpcode::G_GET_ROUNDING:
652 return legalizeGETROUNDING(MI, MRI, Helper);
653 case TargetOpcode::G_SET_ROUNDING:
654 return legalizeSETROUNDING(MI, MRI, Helper);
655 case TargetOpcode::G_GLOBAL_VALUE:
656 return legalizeGLOBAL_VALUE(MI, MRI, Helper);
657 }
658 llvm_unreachable("expected switch to return");
659}
660
661bool X86LegalizerInfo::legalizeSITOFP(MachineInstr &MI,
662 MachineRegisterInfo &MRI,
663 LegalizerHelper &Helper) const {
664 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
665 MachineFunction &MF = *MI.getMF();
666 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
667
668 assert((SrcTy.getSizeInBits() == 16 || SrcTy.getSizeInBits() == 32 ||
669 SrcTy.getSizeInBits() == 64) &&
670 "Unexpected source type for SITOFP in X87 mode.");
671
672 TypeSize MemSize = SrcTy.getSizeInBytes();
673 MachinePointerInfo PtrInfo;
674 Align Alignmt = Helper.getStackTemporaryAlignment(Type: SrcTy);
675 auto SlotPointer = Helper.createStackTemporary(Bytes: MemSize, Alignment: Alignmt, PtrInfo);
676 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
677 PtrInfo, F: MachineMemOperand::MOStore, Size: MemSize, BaseAlignment: Align(MemSize));
678
679 // Store the integer value on the FPU stack.
680 MIRBuilder.buildStore(Val: Src, Addr: SlotPointer, MMO&: *StoreMMO);
681
682 MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
683 PtrInfo, F: MachineMemOperand::MOLoad, Size: MemSize, BaseAlignment: Align(MemSize));
684 MIRBuilder.buildInstr(Opcode: X86::G_FILD)
685 .addDef(RegNo: Dst)
686 .addUse(RegNo: SlotPointer.getReg(Idx: 0))
687 .addMemOperand(MMO: LoadMMO);
688
689 MI.eraseFromParent();
690 return true;
691}
692
693bool X86LegalizerInfo::legalizeFPTOSI(MachineInstr &MI,
694 MachineRegisterInfo &MRI,
695 LegalizerHelper &Helper) const {
696 MachineFunction &MF = *MI.getMF();
697 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
698 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
699
700 TypeSize MemSize = DstTy.getSizeInBytes();
701 MachinePointerInfo PtrInfo;
702 Align Alignmt = Helper.getStackTemporaryAlignment(Type: DstTy);
703 auto SlotPointer = Helper.createStackTemporary(Bytes: MemSize, Alignment: Alignmt, PtrInfo);
704 MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
705 PtrInfo, F: MachineMemOperand::MOStore, Size: MemSize, BaseAlignment: Align(MemSize));
706
707 MIRBuilder.buildInstr(Opcode: X86::G_FIST)
708 .addUse(RegNo: Src)
709 .addUse(RegNo: SlotPointer.getReg(Idx: 0))
710 .addMemOperand(MMO: StoreMMO);
711
712 MIRBuilder.buildLoad(Res: Dst, Addr: SlotPointer, PtrInfo, Alignment: Align(MemSize));
713 MI.eraseFromParent();
714 return true;
715}
716
717bool X86LegalizerInfo::legalizeBuildVector(MachineInstr &MI,
718 MachineRegisterInfo &MRI,
719 LegalizerHelper &Helper) const {
720 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
721 const auto &BuildVector = cast<GBuildVector>(Val&: MI);
722 Register Dst = BuildVector.getReg(Idx: 0);
723 LLT DstTy = MRI.getType(Reg: Dst);
724 MachineFunction &MF = MIRBuilder.getMF();
725 LLVMContext &Ctx = MF.getFunction().getContext();
726 uint64_t DstTySize = DstTy.getScalarSizeInBits();
727
728 SmallVector<Constant *, 4> CstIdxs;
729 for (unsigned i = 0; i < BuildVector.getNumSources(); ++i) {
730 Register Source = BuildVector.getSourceReg(I: i);
731
732 auto ValueAndReg = getIConstantVRegValWithLookThrough(VReg: Source, MRI);
733 if (ValueAndReg) {
734 CstIdxs.emplace_back(Args: ConstantInt::get(Context&: Ctx, V: ValueAndReg->Value));
735 continue;
736 }
737
738 auto FPValueAndReg = getFConstantVRegValWithLookThrough(VReg: Source, MRI);
739 if (FPValueAndReg) {
740 CstIdxs.emplace_back(Args: ConstantFP::get(Context&: Ctx, V: FPValueAndReg->Value));
741 continue;
742 }
743
744 if (getOpcodeDef<GImplicitDef>(Reg: Source, MRI)) {
745 CstIdxs.emplace_back(Args: UndefValue::get(T: Type::getIntNTy(C&: Ctx, N: DstTySize)));
746 continue;
747 }
748 return false;
749 }
750
751 Constant *ConstVal = ConstantVector::get(V: CstIdxs);
752
753 const DataLayout &DL = MIRBuilder.getDataLayout();
754 unsigned AddrSpace = DL.getDefaultGlobalsAddressSpace();
755 Align Alignment(DL.getABITypeAlign(Ty: ConstVal->getType()));
756 auto Addr = MIRBuilder.buildConstantPool(
757 Res: LLT::pointer(AddressSpace: AddrSpace, SizeInBits: DL.getPointerSizeInBits(AS: AddrSpace)),
758 Idx: MF.getConstantPool()->getConstantPoolIndex(C: ConstVal, Alignment));
759 MachineMemOperand *MMO =
760 MF.getMachineMemOperand(PtrInfo: MachinePointerInfo::getConstantPool(MF),
761 f: MachineMemOperand::MOLoad, MemTy: DstTy, base_alignment: Alignment);
762
763 MIRBuilder.buildLoad(Res: Dst, Addr, MMO&: *MMO);
764 MI.eraseFromParent();
765 return true;
766}
767
768bool X86LegalizerInfo::legalizeFPTOUI(MachineInstr &MI,
769 MachineRegisterInfo &MRI,
770 LegalizerHelper &Helper) const {
771 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
772 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
773 unsigned DstSizeInBits = DstTy.getScalarSizeInBits();
774 const LLT s32 = LLT::scalar(SizeInBits: 32);
775 const LLT s64 = LLT::scalar(SizeInBits: 64);
776
777 // Simply reuse FPTOSI when it is possible to widen the type
778 if (DstSizeInBits <= 32) {
779 auto Casted = MIRBuilder.buildFPTOSI(Dst: DstTy == s32 ? s64 : s32, Src0: Src);
780 MIRBuilder.buildTrunc(Res: Dst, Op: Casted);
781 MI.eraseFromParent();
782 return true;
783 }
784
785 return false;
786}
787
788bool X86LegalizerInfo::legalizeUITOFP(MachineInstr &MI,
789 MachineRegisterInfo &MRI,
790 LegalizerHelper &Helper) const {
791 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
792 auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
793 const LLT s32 = LLT::scalar(SizeInBits: 32);
794 const LLT s64 = LLT::scalar(SizeInBits: 64);
795
796 // Simply reuse SITOFP when it is possible to widen the type
797 if (SrcTy.getSizeInBits() <= 32) {
798 auto Ext = MIRBuilder.buildZExt(Res: SrcTy == s32 ? s64 : s32, Op: Src);
799 MIRBuilder.buildSITOFP(Dst, Src0: Ext);
800 MI.eraseFromParent();
801 return true;
802 }
803
804 return false;
805}
806
807bool X86LegalizerInfo::legalizeNarrowingStore(MachineInstr &MI,
808 MachineRegisterInfo &MRI,
809 LegalizerHelper &Helper) const {
810 auto &Store = cast<GStore>(Val&: MI);
811 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
812 MachineMemOperand &MMO = **Store.memoperands_begin();
813 MachineFunction &MF = MIRBuilder.getMF();
814 LLT ValTy = MRI.getType(Reg: Store.getValueReg());
815 auto *NewMMO = MF.getMachineMemOperand(MMO: &MMO, PtrInfo: MMO.getPointerInfo(), Ty: ValTy);
816
817 Helper.Observer.changingInstr(MI&: Store);
818 Store.setMemRefs(MF, MemRefs: {NewMMO});
819 Helper.Observer.changedInstr(MI&: Store);
820 return true;
821}
822
823bool X86LegalizerInfo::legalizeGETROUNDING(MachineInstr &MI,
824 MachineRegisterInfo &MRI,
825 LegalizerHelper &Helper) const {
826 /*
827 The rounding mode is in bits 11:10 of FPSR, and has the following
828 settings:
829 00 Round to nearest
830 01 Round to -inf
831 10 Round to +inf
832 11 Round to 0
833
834 GET_ROUNDING, on the other hand, expects the following:
835 -1 Undefined
836 0 Round to 0
837 1 Round to nearest
838 2 Round to +inf
839 3 Round to -inf
840
841 To perform the conversion, we use a packed lookup table of the four 2-bit
842 values that we can index by FPSP[11:10]
843 0x2d --> (0b00,10,11,01) --> (0,2,3,1) >> FPSR[11:10]
844
845 (0x2d >> ((FPSR >> 9) & 6)) & 3
846 */
847
848 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
849 MachineFunction &MF = MIRBuilder.getMF();
850 Register Dst = MI.getOperand(i: 0).getReg();
851 LLT DstTy = MRI.getType(Reg: Dst);
852 const LLT s8 = LLT::scalar(SizeInBits: 8);
853 const LLT s16 = LLT::scalar(SizeInBits: 16);
854 const LLT s32 = LLT::scalar(SizeInBits: 32);
855
856 // Save FP Control Word to stack slot
857 int MemSize = 2;
858 Align Alignment = Align(2);
859 MachinePointerInfo PtrInfo;
860 auto StackTemp = Helper.createStackTemporary(Bytes: TypeSize::getFixed(ExactSize: MemSize),
861 Alignment, PtrInfo);
862 Register StackPtr = StackTemp.getReg(Idx: 0);
863
864 auto StoreMMO = MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOStore,
865 Size: MemSize, BaseAlignment: Alignment);
866
867 // Store FP Control Word to stack slot using G_FNSTCW16
868 MIRBuilder.buildInstr(Opcode: X86::G_FNSTCW16)
869 .addUse(RegNo: StackPtr)
870 .addMemOperand(MMO: StoreMMO);
871
872 // Load FP Control Word from stack slot
873 auto LoadMMO = MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOLoad,
874 Size: MemSize, BaseAlignment: Alignment);
875
876 auto CWD32 =
877 MIRBuilder.buildZExt(Res: s32, Op: MIRBuilder.buildLoad(Res: s16, Addr: StackPtr, MMO&: *LoadMMO));
878 auto Shifted8 = MIRBuilder.buildTrunc(
879 Res: s8, Op: MIRBuilder.buildLShr(Dst: s32, Src0: CWD32, Src1: MIRBuilder.buildConstant(Res: s8, Val: 9)));
880 auto Masked32 = MIRBuilder.buildZExt(
881 Res: s32, Op: MIRBuilder.buildAnd(Dst: s8, Src0: Shifted8, Src1: MIRBuilder.buildConstant(Res: s8, Val: 6)));
882
883 // LUT is a packed lookup table (0x2d) used to map the 2-bit x87 FPU rounding
884 // mode (from bits 11:10 of the control word) to the values expected by
885 // GET_ROUNDING. The mapping is performed by shifting LUT right by the
886 // extracted rounding mode and masking the result with 3 to obtain the final
887 auto LUT = MIRBuilder.buildConstant(Res: s32, Val: 0x2d);
888 auto LUTShifted = MIRBuilder.buildLShr(Dst: s32, Src0: LUT, Src1: Masked32);
889 auto RetVal =
890 MIRBuilder.buildAnd(Dst: s32, Src0: LUTShifted, Src1: MIRBuilder.buildConstant(Res: s32, Val: 3));
891 auto RetValTrunc = MIRBuilder.buildZExtOrTrunc(Res: DstTy, Op: RetVal);
892
893 MIRBuilder.buildCopy(Res: Dst, Op: RetValTrunc);
894
895 MI.eraseFromParent();
896 return true;
897}
898
899bool X86LegalizerInfo::legalizeSETROUNDING(MachineInstr &MI,
900 MachineRegisterInfo &MRI,
901 LegalizerHelper &Helper) const {
902 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
903 MachineFunction &MF = MIRBuilder.getMF();
904 Register Src = MI.getOperand(i: 0).getReg();
905 const LLT s8 = LLT::scalar(SizeInBits: 8);
906 const LLT s16 = LLT::scalar(SizeInBits: 16);
907 const LLT s32 = LLT::scalar(SizeInBits: 32);
908
909 // Allocate stack slot for control word and MXCSR (4 bytes).
910 int MemSize = 4;
911 Align Alignment = Align(4);
912 MachinePointerInfo PtrInfo;
913 auto StackTemp = Helper.createStackTemporary(Bytes: TypeSize::getFixed(ExactSize: MemSize),
914 Alignment, PtrInfo);
915 Register StackPtr = StackTemp.getReg(Idx: 0);
916
917 auto StoreMMO =
918 MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOStore, Size: 2, BaseAlignment: Align(2));
919 MIRBuilder.buildInstr(Opcode: X86::G_FNSTCW16)
920 .addUse(RegNo: StackPtr)
921 .addMemOperand(MMO: StoreMMO);
922
923 auto LoadMMO =
924 MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOLoad, Size: 2, BaseAlignment: Align(2));
925 auto CWD16 = MIRBuilder.buildLoad(Res: s16, Addr: StackPtr, MMO&: *LoadMMO);
926
927 // Clear RM field (bits 11:10)
928 auto ClearedCWD =
929 MIRBuilder.buildAnd(Dst: s16, Src0: CWD16, Src1: MIRBuilder.buildConstant(Res: s16, Val: 0xf3ff));
930
931 // Check if Src is a constant
932 auto *SrcDef = MRI.getVRegDef(Reg: Src);
933 Register RMBits;
934 Register MXCSRRMBits;
935
936 if (SrcDef && SrcDef->getOpcode() == TargetOpcode::G_CONSTANT) {
937 uint64_t RM = getIConstantFromReg(VReg: Src, MRI).getZExtValue();
938 int FieldVal = X86::getRoundingModeX86(RM);
939
940 if (FieldVal == X86::rmInvalid) {
941 FieldVal = X86::rmToNearest;
942 LLVMContext &C = MF.getFunction().getContext();
943 C.diagnose(DI: DiagnosticInfoUnsupported(
944 MF.getFunction(), "rounding mode is not supported by X86 hardware",
945 DiagnosticLocation(MI.getDebugLoc()), DS_Error));
946 return false;
947 }
948
949 FieldVal = FieldVal << 3;
950 RMBits = MIRBuilder.buildConstant(Res: s16, Val: FieldVal).getReg(Idx: 0);
951 MXCSRRMBits = MIRBuilder.buildConstant(Res: s32, Val: FieldVal).getReg(Idx: 0);
952 } else {
953 // Convert Src (rounding mode) to bits for control word
954 // (0xc9 << (2 * Src + 4)) & 0xc00
955 auto Src32 = MIRBuilder.buildZExtOrTrunc(Res: s32, Op: Src);
956 auto ShiftAmt = MIRBuilder.buildAdd(
957 Dst: s32, Src0: MIRBuilder.buildShl(Dst: s32, Src0: Src32, Src1: MIRBuilder.buildConstant(Res: s32, Val: 1)),
958 Src1: MIRBuilder.buildConstant(Res: s32, Val: 4));
959 auto ShiftAmt8 = MIRBuilder.buildTrunc(Res: s8, Op: ShiftAmt);
960 auto Shifted = MIRBuilder.buildShl(Dst: s16, Src0: MIRBuilder.buildConstant(Res: s16, Val: 0xc9),
961 Src1: ShiftAmt8);
962 RMBits =
963 MIRBuilder.buildAnd(Dst: s16, Src0: Shifted, Src1: MIRBuilder.buildConstant(Res: s16, Val: 0xc00))
964 .getReg(Idx: 0);
965
966 // For non-constant case, we still need to compute MXCSR bits dynamically
967 auto RMBits32 = MIRBuilder.buildZExt(Res: s32, Op: RMBits);
968 MXCSRRMBits =
969 MIRBuilder.buildShl(Dst: s32, Src0: RMBits32, Src1: MIRBuilder.buildConstant(Res: s32, Val: 3))
970 .getReg(Idx: 0);
971 }
972 // Update rounding mode bits
973 auto NewCWD =
974 MIRBuilder.buildOr(Dst: s16, Src0: ClearedCWD, Src1: RMBits, Flags: MachineInstr::Disjoint);
975
976 // Store new FP Control Word to stack
977 auto StoreNewMMO =
978 MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOStore, Size: 2, BaseAlignment: Align(2));
979 MIRBuilder.buildStore(Val: NewCWD, Addr: StackPtr, MMO&: *StoreNewMMO);
980
981 // Load FP control word from the slot using G_FLDCW16
982 auto LoadNewMMO =
983 MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOLoad, Size: 2, BaseAlignment: Align(2));
984 MIRBuilder.buildInstr(Opcode: X86::G_FLDCW16)
985 .addUse(RegNo: StackPtr)
986 .addMemOperand(MMO: LoadNewMMO);
987
988 if (Subtarget.hasSSE1()) {
989 // Store MXCSR to stack (use STMXCSR)
990 auto StoreMXCSRMMO = MF.getMachineMemOperand(
991 PtrInfo, F: MachineMemOperand::MOStore, Size: 4, BaseAlignment: Align(4));
992 MIRBuilder.buildInstr(Opcode: TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS)
993 .addIntrinsicID(ID: Intrinsic::x86_sse_stmxcsr)
994 .addUse(RegNo: StackPtr)
995 .addMemOperand(MMO: StoreMXCSRMMO);
996
997 // Load MXCSR from stack
998 auto LoadMXCSRMMO = MF.getMachineMemOperand(
999 PtrInfo, F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: Align(4));
1000 auto MXCSR = MIRBuilder.buildLoad(Res: s32, Addr: StackPtr, MMO&: *LoadMXCSRMMO);
1001
1002 // Clear RM field (bits 14:13)
1003 auto ClearedMXCSR = MIRBuilder.buildAnd(
1004 Dst: s32, Src0: MXCSR, Src1: MIRBuilder.buildConstant(Res: s32, Val: 0xffff9fff));
1005
1006 // Update rounding mode bits
1007 auto NewMXCSR = MIRBuilder.buildOr(Dst: s32, Src0: ClearedMXCSR, Src1: MXCSRRMBits);
1008
1009 // Store new MXCSR to stack
1010 auto StoreNewMXCSRMMO = MF.getMachineMemOperand(
1011 PtrInfo, F: MachineMemOperand::MOStore, Size: 4, BaseAlignment: Align(4));
1012 MIRBuilder.buildStore(Val: NewMXCSR, Addr: StackPtr, MMO&: *StoreNewMXCSRMMO);
1013
1014 // Load MXCSR from stack (use LDMXCSR)
1015 auto LoadNewMXCSRMMO = MF.getMachineMemOperand(
1016 PtrInfo, F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: Align(4));
1017 MIRBuilder.buildInstr(Opcode: TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS)
1018 .addIntrinsicID(ID: Intrinsic::x86_sse_ldmxcsr)
1019 .addUse(RegNo: StackPtr)
1020 .addMemOperand(MMO: LoadNewMXCSRMMO);
1021 }
1022
1023 MI.eraseFromParent();
1024 return true;
1025}
1026
1027bool X86LegalizerInfo::legalizeGLOBAL_VALUE(MachineInstr &MI,
1028 MachineRegisterInfo &MRI,
1029 LegalizerHelper &Helper) const {
1030 const GlobalValue *GV = MI.getOperand(i: 1).getGlobal();
1031 Register Dst = MI.getOperand(i: 0).getReg();
1032 LLT DstTy = MRI.getType(Reg: Dst);
1033 unsigned GVOpFlags = Subtarget.classifyGlobalReference(GV);
1034
1035 // For stub references (GOT/PLT), we need G_WRAPPER_RIP + load
1036 if (isGlobalStubReference(TargetFlag: GVOpFlags)) {
1037 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1038 MachineFunction &MF = MIRBuilder.getMF();
1039
1040 Register StubAddr = MRI.createGenericVirtualRegister(Ty: DstTy);
1041 MIRBuilder.buildInstr(Opcode: X86::G_WRAPPER_RIP)
1042 .addDef(RegNo: StubAddr)
1043 .addGlobalAddress(GV);
1044
1045 MachineMemOperand *MMO = MF.getMachineMemOperand(
1046 PtrInfo: MachinePointerInfo::getGOT(MF), f: MachineMemOperand::MOLoad, MemTy: DstTy,
1047 base_alignment: Align(DstTy.getSizeInBytes()));
1048 MIRBuilder.buildLoad(Res: Dst, Addr: StubAddr, MMO&: *MMO);
1049 MI.eraseFromParent();
1050 }
1051 return true;
1052}
1053
1054bool X86LegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
1055 MachineInstr &MI) const {
1056 return true;
1057}
1058